[{"id":"IL-0109","title":{"en":"A retried node rollover on a DFDS EKS cluster set the Auto Scaling group back to 15 instances, and the default termination policy removed every healthy worker node, leaving the cluster with no nodes","ko":"로그 수집을 고치려던 EKS 노드 롤오버가 꼬이면서 Auto Scaling 그룹이 정상 워커 노드를 모두 종료해 클러스터에 노드가 하나도 남지 않았다."},"org":"DFDS","org_slug":"dfds","date":"2023-02-14","severity":"sev1","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0109","shared_components":["launch-template","terraform"]},{"id":"IL-0105","title":{"en":"Replacing the AWS VPC CNI during a Terraform refactoring detached the ALB target groups, and every service in the Kubernetes cluster returned 503","ko":"VPC CNI 교체로 ALB 에서 타깃 그룹이 떨어져 나가 쿠버네티스 클러스터의 모든 서비스가 503 을 냈다"},"org":"DFDS","org_slug":"dfds","date":"2021-06-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0105","shared_components":["terraform","cloud-load-balancer"]},{"id":"IL-0346","title":{"en":"Render's routing service returned intermittent 404s for web services and static sites in Oregon","ko":"Oregon 리전의 웹 서비스와 정적 사이트가 간헐적으로 404 응답을 냈다"},"org":"Render","org_slug":"render","date":"2025-11-05","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0346","shared_components":["cloud-load-balancer"]},{"id":"IL-0543","title":{"en":"Twingate API traffic piled onto one backend behind the global load balancer, failing core services for 29 minutes","ko":"Twingate 의 인증과 관리 서비스 전반이 29분 동안 오류를 내며 멈췄다"},"org":"Twingate","org_slug":"twingate","date":"2025-05-28","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0543","shared_components":["cloud-load-balancer"]},{"id":"IL-0192","title":{"en":"A Crossplane composition merge-policy change for a TLS policy update switched Kubernetes Services from LoadBalancer to ClusterIP and deleted the load balancers behind about 25% of Grafana Cloud services","ko":"Crossplane 컴포지션의 병합 정책 변경이 Service 타입을 ClusterIP 로 바꿔 클라우드 로드 밸런서가 지워지고 Grafana Cloud 서비스의 약 25%가 끊겼다."},"org":"Grafana Labs","org_slug":"grafana-labs","date":"2025-02-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0192","shared_components":["cloud-load-balancer"]},{"id":"IL-0355","title":{"en":"Railway's Google Cloud edge stopped routing inbound requests to apps in all regions for about 10 minutes","ko":"Google Cloud 위의 Railway 에지 서버로 들어오는 트래픽이 약 10분 동안 모두 막혔다"},"org":"Railway","org_slug":"railway","date":"2024-12-17","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0355","shared_components":["terraform"]},{"id":"IL-0019","title":{"en":"An EKS Terraform module upgrade shrank node root disks from 100 GB to 20 GB and image pulls failed with no space left on device","ko":"EKS 모듈 18 업그레이드로 새 노드의 디스크가 20Gb 로 줄어 이미지 풀이 no space left on device 로 실패했다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-09-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0019","shared_components":["launch-template"]},{"id":"IL-0473","title":{"en":"Jenkins infrastructure went fully down after a pod IP range fix deleted every AKS node pool and could not recreate them","ko":"Azure 공개 쿠버네티스 클러스터의 노드 풀이 모두 지워지고 다시 만들어지지 않아 Jenkins 인프라 서비스가 4시간 넘게 멈췄다"},"org":"Jenkins project","org_slug":"jenkins-project","date":"2023-07-07","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0473","shared_components":["cloud-load-balancer"]},{"id":"IL-0091","title":{"en":"Changing the worker node instance type through Terraform recycled all nodes at once and deleted users' production pods together","ko":"Terraform 으로 노드 인스턴스 타입을 바꾸자 노드가 한꺼번에 교체되어 사용자 운영 파드가 모두 지워지고 서비스가 내려갔다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-06-06","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0091","shared_components":["terraform"]},{"id":"IL-0541","title":{"en":"Two Twingate Relay clusters were discoverable but unreachable, so Clients and Connectors could not attach to them","ko":"Twingate 의 us-east4 와 europe-west6 Relay 클러스터에 Client 와 Connector 가 접속하지 못했다"},"org":"Twingate","org_slug":"twingate","date":"2023-02-14","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0541","shared_components":["terraform"]},{"id":"IL-0363","title":{"en":"static.crates.io did not resolve for 10-15 minutes during a Terraform deployment, so Rust builds could not fetch crates","ko":"static.crates.io 의 DNS 레코드가 10~15분 동안 해석되지 않아 crate 를 받지 못했다"},"org":"Rust Project","org_slug":"rust-project","date":"2023-01-25","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0363","shared_components":["terraform"]},{"id":"IL-0513","title":{"en":"PyPI backends on Kubernetes were unreachable over HTTPS for 20 minutes when the Ingress ELB lost its HTTPS listener","ko":"쿠버네티스 노드 순차 재시작 중 Ingress 가 관리하는 ELB 의 HTTPS 리스너가 꺼져 PyPI 백엔드에 HTTPS 로 닿지 못했다"},"org":"Python Software Foundation","org_slug":"python-software-foundation","date":"2022-10-24","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0513","shared_components":["cloud-load-balancer"]},{"id":"IL-0177","title":{"en":"WebSocket connections to LINE's messaging-hub on Kubernetes dropped all at once whenever another service's change reloaded a shared HAProxy L7 load balancer, and the reconnect wave overloaded a token API and filled nf_conntrack on Nginx","ko":"다른 팀의 설정 배포로 공유 L7 HAProxy 가 재로드될 때마다 messaging-hub 의 웹소켓 연결이 한꺼번에 끊겼다."},"org":"LINE Plus","org_slug":"line-plus","date":"2022-10-14","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0177","shared_components":["cloud-load-balancer"]},{"id":"IL-0208","title":{"en":"A GitLab chart bump that renamed Kubernetes Service and Deployment ports made the GCP load balancer reconfigure, and GitLab.com slowed down and returned intermittent 50x errors","ko":"GitLab Helm 차트 업그레이드가 Service 와 Deployment 의 포트 이름을 바꿔 GCP 로드 밸런서가 다시 구성되는 동안 GitLab.com 요청이 느려지고 50x 오류가 났다"},"org":"GitLab","org_slug":"gitlab","date":"2022-03-31","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0208","shared_components":["cloud-load-balancer"]},{"id":"IL-0111","title":{"en":"Switching the main Kubernetes ingress controller from Traefik v1 to v2 stopped X-Forwarded headers being passed, which broke the Okta login flow for every service using Okta and stopped DFDS online bookings for over an hour","ko":"메인 인그레스 컨트롤러를 Traefik v2 로 바꾸자 X-Forwarded 헤더가 전달되지 않아 Okta 로그인이 깨졌고, DFDS 온라인 예약이 한 시간 넘게 멈췄다"},"org":"DFDS","org_slug":"dfds","date":"2021-08-26","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0111","shared_components":["cloud-load-balancer"]},{"id":"IL-0061","title":{"en":"A Terraform pull request merged out of order deleted a production GKE cluster, and recreating it deleted a second one","ko":"Terraform 상태 파일이 리뷰 빌드에서 바뀐 데다 권한 차이까지 겹쳐 Terraform 이 운영 GKE 클러스터 세 개 중 두 개를 삭제했다"},"org":"Spotify","org_slug":"spotify","date":"2019-05-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0061","shared_components":["terraform"]},{"id":"IL-0052","title":{"en":"A cryptocurrency miner ran on every node of the development and staging clusters through a public, unauthenticated Weave Scope dashboard","ko":"인증 없이 인터넷에 열린 Weave Scope 대시보드를 통해 암호화폐 채굴기가 개발 클러스터와 스테이징 클러스터의 모든 노드에서 돌았다."},"org":"JW Player","org_slug":"jw-player","date":"2019-03-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0052","shared_components":["cloud-load-balancer"]}]