[{"id":"IL-0346","title":{"en":"Render's routing service returned intermittent 404s for web services and static sites in Oregon","ko":"Oregon 리전의 웹 서비스와 정적 사이트가 간헐적으로 404 응답을 냈다"},"org":"Render","org_slug":"render","date":"2025-11-05","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0346","shared_components":["cloud-load-balancer"]},{"id":"IL-0543","title":{"en":"Twingate API traffic piled onto one backend behind the global load balancer, failing core services for 29 minutes","ko":"Twingate 의 인증과 관리 서비스 전반이 29분 동안 오류를 내며 멈췄다"},"org":"Twingate","org_slug":"twingate","date":"2025-05-28","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0543","shared_components":["cloud-load-balancer"]},{"id":"IL-0192","title":{"en":"A Crossplane composition merge-policy change for a TLS policy update switched Kubernetes Services from LoadBalancer to ClusterIP and deleted the load balancers behind about 25% of Grafana Cloud services","ko":"Crossplane 컴포지션의 병합 정책 변경이 Service 타입을 ClusterIP 로 바꿔 클라우드 로드 밸런서가 지워지고 Grafana Cloud 서비스의 약 25%가 끊겼다."},"org":"Grafana Labs","org_slug":"grafana-labs","date":"2025-02-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0192","shared_components":["cloud-load-balancer"]},{"id":"IL-0114","title":{"en":"An unpinned AWS VPC CNI plugin was updated automatically during a capacity expansion, leaving new pods Pending without IP addresses and stopping CoW Protocol settlements for about 3.5 hours","ko":"클러스터 용량을 늘리던 중 버전이 고정되지 않은 VPC CNI 플러그인이 자동으로 업데이트돼 새 파드가 IP 를 받지 못하고 Pending 에 머물렀고, 메인넷 운영 환경의 정산이 약 3.5시간 멈췄다."},"org":"CoW Protocol","org_slug":"cow-protocol","date":"2024-04-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0114","shared_components":["node-pool"]},{"id":"IL-0019","title":{"en":"An EKS Terraform module upgrade shrank node root disks from 100 GB to 20 GB and image pulls failed with no space left on device","ko":"EKS 모듈 18 업그레이드로 새 노드의 디스크가 20Gb 로 줄어 이미지 풀이 no space left on device 로 실패했다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-09-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0019","shared_components":["node-pool"]},{"id":"IL-0017","title":{"en":"AWS VPC CNI stopped allocating pod IP addresses on six nodes of the live cluster, leaving new pods in ContainerCreating","ko":"live 클러스터 노드 6대에서 VPC CNI 가 IP 주소를 할당하지 못해 새 파드가 ContainerCreating 에 멈췄다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-07-21","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0017","shared_components":["node-pool"]},{"id":"IL-0091","title":{"en":"Changing the worker node instance type through Terraform recycled all nodes at once and deleted users' production pods together","ko":"Terraform 으로 노드 인스턴스 타입을 바꾸자 노드가 한꺼번에 교체되어 사용자 운영 파드가 모두 지워지고 서비스가 내려갔다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-06-06","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0091","shared_components":["node-pool"]},{"id":"IL-0231","title":{"en":"Deleting EKS managed node groups also deleted Karpenter's instance profile, and every Stytch worker node shut down","ko":"EKS 클러스터의 워커 노드가 모두 꺼지고 새 노드가 뜨지 않아 Stytch API 가 14분 동안 완전히 멈췄다"},"org":"Stytch","org_slug":"stytch","date":"2023-02-23","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0231","shared_components":["node-pool"]},{"id":"IL-0109","title":{"en":"A retried node rollover on a DFDS EKS cluster set the Auto Scaling group back to 15 instances, and the default termination policy removed every healthy worker node, leaving the cluster with no nodes","ko":"로그 수집을 고치려던 EKS 노드 롤오버가 꼬이면서 Auto Scaling 그룹이 정상 워커 노드를 모두 종료해 클러스터에 노드가 하나도 남지 않았다."},"org":"DFDS","org_slug":"dfds","date":"2023-02-14","severity":"sev1","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0109","shared_components":["node-pool"]},{"id":"IL-0116","title":{"en":"A CNI configuration change incompatible with a hardware variant that exists only in production cut network connectivity in an OpenAI GPU cluster and took text-davinci-003 fully down","ko":"CNI 설정 변경이 스테이징에 없던 하드웨어 종류와 호환되지 않아 GPU 클러스터 하나의 워크로드가 네트워크 연결을 잃었고, text-davinci-003 모델이 완전히 멈췄다."},"org":"OpenAI","org_slug":"openai","date":"2023-01-25","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0116","shared_components":["cni-plugin"]},{"id":"IL-0513","title":{"en":"PyPI backends on Kubernetes were unreachable over HTTPS for 20 minutes when the Ingress ELB lost its HTTPS listener","ko":"쿠버네티스 노드 순차 재시작 중 Ingress 가 관리하는 ELB 의 HTTPS 리스너가 꺼져 PyPI 백엔드에 HTTPS 로 닿지 못했다"},"org":"Python Software Foundation","org_slug":"python-software-foundation","date":"2022-10-24","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0513","shared_components":["cloud-load-balancer"]},{"id":"IL-0177","title":{"en":"WebSocket connections to LINE's messaging-hub on Kubernetes dropped all at once whenever another service's change reloaded a shared HAProxy L7 load balancer, and the reconnect wave overloaded a token API and filled nf_conntrack on Nginx","ko":"다른 팀의 설정 배포로 공유 L7 HAProxy 가 재로드될 때마다 messaging-hub 의 웹소켓 연결이 한꺼번에 끊겼다."},"org":"LINE Plus","org_slug":"line-plus","date":"2022-10-14","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0177","shared_components":["cloud-load-balancer"]},{"id":"IL-0208","title":{"en":"A GitLab chart bump that renamed Kubernetes Service and Deployment ports made the GCP load balancer reconfigure, and GitLab.com slowed down and returned intermittent 50x errors","ko":"GitLab Helm 차트 업그레이드가 Service 와 Deployment 의 포트 이름을 바꿔 GCP 로드 밸런서가 다시 구성되는 동안 GitLab.com 요청이 느려지고 50x 오류가 났다"},"org":"GitLab","org_slug":"gitlab","date":"2022-03-31","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0208","shared_components":["cloud-load-balancer"]},{"id":"IL-0111","title":{"en":"Switching the main Kubernetes ingress controller from Traefik v1 to v2 stopped X-Forwarded headers being passed, which broke the Okta login flow for every service using Okta and stopped DFDS online bookings for over an hour","ko":"메인 인그레스 컨트롤러를 Traefik v2 로 바꾸자 X-Forwarded 헤더가 전달되지 않아 Okta 로그인이 깨졌고, DFDS 온라인 예약이 한 시간 넘게 멈췄다"},"org":"DFDS","org_slug":"dfds","date":"2021-08-26","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0111","shared_components":["cloud-load-balancer"]},{"id":"IL-0105","title":{"en":"Replacing the AWS VPC CNI during a Terraform refactoring detached the ALB target groups, and every service in the Kubernetes cluster returned 503","ko":"VPC CNI 교체로 ALB 에서 타깃 그룹이 떨어져 나가 쿠버네티스 클러스터의 모든 서비스가 503 을 냈다"},"org":"DFDS","org_slug":"dfds","date":"2021-06-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0105","shared_components":["cloud-load-balancer"]},{"id":"IL-0365","title":{"en":"A Terraform change to auto-scaling groups detached all worker nodes from both ALBs and cut off every cluster service","ko":"ALB 의 인스턴스 연결이 모두 사라져 Hellman 쿠버네티스 클러스터의 모든 서비스가 외부에서 닿지 않았다"},"org":"DFDS","org_slug":"dfds","date":"2020-06-09","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0365","shared_components":["cloud-load-balancer"]},{"id":"IL-0052","title":{"en":"A cryptocurrency miner ran on every node of the development and staging clusters through a public, unauthenticated Weave Scope dashboard","ko":"인증 없이 인터넷에 열린 Weave Scope 대시보드를 통해 암호화폐 채굴기가 개발 클러스터와 스테이징 클러스터의 모든 노드에서 돌았다."},"org":"JW Player","org_slug":"jw-player","date":"2019-03-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0052","shared_components":["cloud-load-balancer"]},{"id":"IL-0055","title":{"en":"A failing CronJob with the default concurrency policy piled up thousands of restarting pods and took down a cluster","ko":"실패하는 CronJob 이 기본 concurrencyPolicy 때문에 겹겹이 쌓여 재시작하는 파드가 수천 개로 늘었고 클러스터가 내려갔다"},"org":"Target","org_slug":"target","date":"2018-09-20","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0055","shared_components":["node-pool"]}]