[{"id":"IL-0179","title":{"en":"With DEBUG logging and synchronous log output, a high-traffic service filled the 64 KB stdout pipe, so request threads blocked in write() while holding database connections and HikariCP ran dry, failing requests in 30-second bursts","ko":"DEBUG 로그가 쏟아지자 동기 로깅을 하던 요청 스레드들이 가득 찬 stdout 파이프에서 멈춘 채 DB 커넥션을 쥐고 있어, HikariCP 커넥션 풀이 고갈되고 요청이 30초 주기로 한꺼번에 실패했다."},"org":"MUSINSA","org_slug":"musinsa","date":"2026-10-01","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0179","shared_components":["container-runtime","telemetry-agent"]},{"id":"IL-0019","title":{"en":"An EKS Terraform module upgrade shrank node root disks from 100 GB to 20 GB and image pulls failed with no space left on device","ko":"EKS 모듈 18 업그레이드로 새 노드의 디스크가 20Gb 로 줄어 이미지 풀이 no space left on device 로 실패했다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-09-18","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0019","shared_components":["launch-template","node-pool"]},{"id":"IL-0091","title":{"en":"Changing the worker node instance type through Terraform recycled all nodes at once and deleted users' production pods together","ko":"Terraform 으로 노드 인스턴스 타입을 바꾸자 노드가 한꺼번에 교체되어 사용자 운영 파드가 모두 지워지고 서비스가 내려갔다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-06-06","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0091","shared_components":["node-pool","terraform"]},{"id":"IL-0005","title":{"en":"A new telemetry service overloaded the Kubernetes API servers of large clusters and broke DNS-based service discovery for all OpenAI services","ko":"새 텔레메트리 서비스가 모든 노드에서 무거운 쿠버네티스 API 요청을 보내 대형 클러스터의 컨트롤 플레인이 무너졌고, DNS 기반 서비스 디스커버리가 깨져 OpenAI 의 모든 서비스가 멈췄다"},"org":"OpenAI","org_slug":"openai","date":"2024-12-11","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0005","shared_components":["telemetry-agent"]},{"id":"IL-0114","title":{"en":"An unpinned AWS VPC CNI plugin was updated automatically during a capacity expansion, leaving new pods Pending without IP addresses and stopping CoW Protocol settlements for about 3.5 hours","ko":"클러스터 용량을 늘리던 중 버전이 고정되지 않은 VPC CNI 플러그인이 자동으로 업데이트돼 새 파드가 IP 를 받지 못하고 Pending 에 머물렀고, 메인넷 운영 환경의 정산이 약 3.5시간 멈췄다."},"org":"CoW Protocol","org_slug":"cow-protocol","date":"2024-04-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0114","shared_components":["node-pool"]},{"id":"IL-0017","title":{"en":"AWS VPC CNI stopped allocating pod IP addresses on six nodes of the live cluster, leaving new pods in ContainerCreating","ko":"live 클러스터 노드 6대에서 VPC CNI 가 IP 주소를 할당하지 못해 새 파드가 ContainerCreating 에 멈췄다"},"org":"UK Ministry of Justice","org_slug":"uk-ministry-of-justice","date":"2023-07-21","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0017","shared_components":["node-pool"]},{"id":"IL-0105","title":{"en":"Replacing the AWS VPC CNI during a Terraform refactoring detached the ALB target groups, and every service in the Kubernetes cluster returned 503","ko":"VPC CNI 교체로 ALB 에서 타깃 그룹이 떨어져 나가 쿠버네티스 클러스터의 모든 서비스가 503 을 냈다"},"org":"DFDS","org_slug":"dfds","date":"2021-06-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0105","shared_components":["terraform"]},{"id":"IL-0061","title":{"en":"A Terraform pull request merged out of order deleted a production GKE cluster, and recreating it deleted a second one","ko":"Terraform 상태 파일이 리뷰 빌드에서 바뀐 데다 권한 차이까지 겹쳐 Terraform 이 운영 GKE 클러스터 세 개 중 두 개를 삭제했다"},"org":"Spotify","org_slug":"spotify","date":"2019-05-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0061","shared_components":["terraform"]},{"id":"IL-0055","title":{"en":"A failing CronJob with the default concurrency policy piled up thousands of restarting pods and took down a cluster","ko":"실패하는 CronJob 이 기본 concurrencyPolicy 때문에 겹겹이 쌓여 재시작하는 파드가 수천 개로 늘었고 클러스터가 내려갔다"},"org":"Target","org_slug":"target","date":"2018-09-20","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0055","shared_components":["node-pool"]}]