[{"id":"IL-0077","title":{"en":"EKS cluster went read-only for over seven hours after Kyverno ephemeralreports filled the etcd database past its size limit","ko":"Kyverno 1.12 가 만든 ephemeralreports 객체가 쌓여 EKS 의 etcd 데이터베이스가 한도를 넘었고 운영 클러스터가 읽기 전용이 되어 모든 쓰기가 거부됐다"},"org":"Marcin Cuber","org_slug":"marcin-cuber","date":"2024-06-05","severity":"sev1","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0077","shared_components":["kube-apiserver","etcd"]},{"id":"IL-0072","title":{"en":"High etcd write latency made the API servers return stale data, and the production cluster flipped between two views of its state","ko":"etcd 쓰기 지연으로 API 서버가 오래된 데이터를 읽어 클러스터 상태가 두 가지로 번갈아 보였고, 주 운영 클러스터 전체가 약 4시간 멈췄다"},"org":"Nordstrom","org_slug":"nordstrom","date":"2017-12-15","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0072","shared_components":["kube-apiserver","etcd"]},{"id":"IL-0138","title":{"en":"A failed Nvidia driver build left broken kernel packages on the only Kubernetes master node, which hit a kernel panic on reboot after a power outage and took the homelab cluster down for six days","ko":"nvidia-dkms-390 드라이버 빌드 실패로 망가진 커널 헤더가 몇 주 동안 방치됐다가, 정전 뒤 재부팅하자 하나뿐인 마스터 노드가 커널 패닉에 빠져 홈랩 쿠버네티스 클러스터가 6일 동안 멈췄다."},"org":"Barush Mendez","org_slug":"barush-mendez","date":"2025-03-03","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0138","shared_components":["kube-apiserver"]},{"id":"IL-0112","title":{"en":"The daily node rollover put DFDS's EKS nodes on an AMI later recalled for a kernel issue, causing \"PLEG is not healthy\" NodeNotReady flaps and pod network timeouts","ko":"나중에 회수된 EKS 노드 AMI 로 노드를 교체한 뒤 노드가 PLEG is not healthy 를 보고하며 NotReady 를 오갔고 파드 네트워크 타임아웃이 났다"},"org":"DFDS","org_slug":"dfds","date":"2022-10-31","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0112","shared_components":["kubelet"]},{"id":"IL-0139","title":{"en":"Pods on GKE clusters with Calico network policy got stuck Terminating or Pending after upgrading to 1.22 or later because of a race condition in the Calico component","ko":"Calico 네트워크 정책을 쓰는 GKE 클러스터를 1.22 이상으로 업그레이드하자 경쟁 조건 때문에 CNI 정리가 Unauthorized 로 실패해 파드가 Terminating 이나 Pending 에 멈췄다."},"org":"Google Cloud","org_slug":"google-cloud","date":"2022-09-15","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0139","shared_components":["kubelet"]},{"id":"IL-0190","title":{"en":"Cortex distributors kept a stuck connection to a dead etcd leader after a Kubernetes upgrade terminated its node, failing writes to Hosted Prometheus for 12 minutes","ko":"쿠버네티스 업그레이드로 etcd 리더가 있던 노드가 갑자기 종료된 뒤, Cortex distributor 두 개가 옛 리더와 맺은 TCP 연결이 멈춘 채 그 연결을 계속 써서 런던 리전 Hosted Prometheus 쓰기가 12분 동안 일부 실패했다."},"org":"Grafana Labs","org_slug":"grafana-labs","date":"2020-03-16","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0190","shared_components":["etcd"]},{"id":"IL-0012","title":{"en":"CPU limits and an unpatched kernel CFS quota bug throttled multi-threaded services into stalls, failed health checks and timeouts","ko":"CPU limit 과 커널 CFS 쿼터 버그로 파드가 과하게 스로틀링돼 헬스 체크가 실패하고 애플리케이션이 멈췄다"},"org":"Omio","org_slug":"omio","date":"2020-02-17","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0012","shared_components":["kubelet"]},{"id":"IL-0030","title":{"en":"Most nodes of a cluster went NotReady together for 15 minutes while the API server sat behind an AWS ELB","ko":"쿠버네티스 v1.7.4 클러스터에서 노드들이 한꺼번에 NotReady 가 되어 쓸 수 있는 노드가 20% 미만으로 줄었다."},"org":"Zalando","org_slug":"zalando","date":"2019-02-11","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0030","shared_components":["kube-apiserver"]},{"id":"IL-0074","title":{"en":"All AirMap platform services down as worker nodes went NotReady and pods crash-looped on network and DNS failures","ko":"워커 노드가 잇달아 Not Ready 에 빠지고 네트워크와 DNS 실패로 사이드카가 죽으면서, AirMap 플랫폼의 모든 서비스가 다섯 시간 동안 응답하지 않았다."},"org":"AirMap","org_slug":"airmap","date":"2018-07-19","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0074","shared_components":["kubelet"]},{"id":"IL-0038","title":{"en":"kubelet rollout hit a Kubernetes bug and left every node in a cluster NotReady, triggering mass pod eviction","ko":"kubelet 설정 변경을 롤링 업데이트하다 쿠버네티스 버그로 kubelet 이 올라오지 못해 한 클러스터의 노드가 NotReady 가 되고 파드 절반쯤이 축출됐다"},"org":"Oath","org_slug":"oath","date":"2018-05-04","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0038","shared_components":["kubelet"]}]