[{"id":"IL-0192","title":{"en":"A Crossplane composition merge-policy change for a TLS policy update switched Kubernetes Services from LoadBalancer to ClusterIP and deleted the load balancers behind about 25% of Grafana Cloud services","ko":"Crossplane 컴포지션의 병합 정책 변경이 Service 타입을 ClusterIP 로 바꿔 클라우드 로드 밸런서가 지워지고 Grafana Cloud 서비스의 약 25%가 끊겼다."},"org":"Grafana Labs","org_slug":"grafana-labs","date":"2025-02-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0192","shared_components":["cloud-load-balancer"]},{"id":"IL-0114","title":{"en":"An unpinned AWS VPC CNI plugin was updated automatically during a capacity expansion, leaving new pods Pending without IP addresses and stopping CoW Protocol settlements for about 3.5 hours","ko":"클러스터 용량을 늘리던 중 버전이 고정되지 않은 VPC CNI 플러그인이 자동으로 업데이트돼 새 파드가 IP 를 받지 못하고 Pending 에 머물렀고, 메인넷 운영 환경의 정산이 약 3.5시간 멈췄다."},"org":"CoW Protocol","org_slug":"cow-protocol","date":"2024-04-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0114","shared_components":["vpc-cni"]},{"id":"IL-0019","title":{"en":"An EKS Terraform module upgrade shrank node root disks from 100 GB to 20 GB and image pulls failed with no space left on device","ko":"EKS 모듈 18 업그레이드로 새 노드의 디스크가 20Gb 로 줄어 이미지 풀이 no space left on device 로 실패했다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-09-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0019","shared_components":["vpc-cni"]},{"id":"IL-0017","title":{"en":"AWS VPC CNI stopped allocating pod IP addresses on six nodes of the live cluster, leaving new pods in ContainerCreating","ko":"live 클러스터 노드 6대에서 VPC CNI 가 IP 주소를 할당하지 못해 새 파드가 ContainerCreating 에 멈췄다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-07-21","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0017","shared_components":["vpc-cni"]},{"id":"IL-0091","title":{"en":"Changing the worker node instance type through Terraform recycled all nodes at once and deleted users' production pods together","ko":"Terraform 으로 노드 인스턴스 타입을 바꾸자 노드가 한꺼번에 교체되어 사용자 운영 파드가 모두 지워지고 서비스가 내려갔다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-06-06","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0091","shared_components":["terraform"]},{"id":"IL-0109","title":{"en":"A retried node rollover on a DFDS EKS cluster set the Auto Scaling group back to 15 instances, and the default termination policy removed every healthy worker node, leaving the cluster with no nodes","ko":"로그 수집을 고치려던 EKS 노드 롤오버가 꼬이면서 Auto Scaling 그룹이 정상 워커 노드를 모두 종료해 클러스터에 노드가 하나도 남지 않았다."},"org":"DFDS","org_slug":"dfds","date":"2023-02-14","severity":"sev1","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0109","shared_components":["terraform"]},{"id":"IL-0177","title":{"en":"WebSocket connections to LINE's messaging-hub on Kubernetes dropped all at once whenever another service's change reloaded a shared HAProxy L7 load balancer, and the reconnect wave overloaded a token API and filled nf_conntrack on Nginx","ko":"다른 팀의 설정 배포로 공유 L7 HAProxy 가 재로드될 때마다 messaging-hub 의 웹소켓 연결이 한꺼번에 끊겼다."},"org":"LINE+","org_slug":"line","date":"2022-10-14","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0177","shared_components":["cloud-load-balancer"]},{"id":"IL-0208","title":{"en":"A GitLab chart bump that renamed Kubernetes Service and Deployment ports made the GCP load balancer reconfigure, and GitLab.com slowed down and returned intermittent 50x errors","ko":"GitLab Helm 차트 업그레이드가 Service 와 Deployment 의 포트 이름을 바꿔 GCP 로드 밸런서가 다시 구성되는 동안 GitLab.com 요청이 느려지고 50x 오류가 났다"},"org":"GitLab","org_slug":"gitlab","date":"2022-03-31","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0208","shared_components":["cloud-load-balancer"]},{"id":"IL-0111","title":{"en":"Switching the main Kubernetes ingress controller from Traefik v1 to v2 stopped X-Forwarded headers being passed, which broke the Okta login flow for every service using Okta and stopped DFDS online bookings for over an hour","ko":"메인 인그레스 컨트롤러를 Traefik v2 로 바꾸자 X-Forwarded 헤더가 전달되지 않아 Okta 로그인이 깨졌고, DFDS 온라인 예약이 한 시간 넘게 멈췄다"},"org":"DFDS","org_slug":"dfds","date":"2021-08-26","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0111","shared_components":["cloud-load-balancer"]},{"id":"IL-0061","title":{"en":"A Terraform pull request merged out of order deleted a production GKE cluster, and recreating it deleted a second one","ko":"Terraform 상태 파일이 리뷰 빌드에서 바뀐 데다 권한 차이까지 겹쳐 Terraform 이 운영 GKE 클러스터 세 개 중 두 개를 삭제했다"},"org":"Spotify","org_slug":"spotify","date":"2019-05-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0061","shared_components":["terraform"]},{"id":"IL-0052","title":{"en":"A cryptocurrency miner ran on every node of the development and staging clusters through a public, unauthenticated Weave Scope dashboard","ko":"인증 없이 인터넷에 열린 Weave Scope 대시보드를 통해 암호화폐 채굴기가 개발 클러스터와 스테이징 클러스터의 모든 노드에서 돌았다."},"org":"JW Player","org_slug":"jw-player","date":"2019-03-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0052","shared_components":["cloud-load-balancer"]}]