[{"id":"IL-0104","title":{"en":"After a CoreDNS pod was OOMKilled, the talk names three causes: Istio's aggressive DNS cache refresh on one reused client port, kube-proxy not cleaning up conntrack entries of terminated pods, and the 2-minute UDP conntrack timeout","ko":"Istio 의 공격적인 DNS 캐시 갱신과 클라이언트 포트 재사용, kube-proxy 가 종료된 파드의 conntrack 항목을 정리하지 않는 문제, 2분짜리 UDP conntrack 기본 타임아웃이 겹쳐 CoreDNS 파드가 OOMKilled 됐다."},"org":"Cisco ThousandEyes","org_slug":"cisco-thousandeyes","date":"2025-09-04","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0104","shared_components":["conntrack"]},{"id":"IL-0247","title":{"en":"One in a million search requests took 100 times longer on Pinterest's Manas leaf nodes after moving to Kubernetes","ko":"쿠버네티스로 옮긴 검색 시스템 Manas 에서 요청 백만 건에 한 건이 평소보다 100배 느려졌다"},"org":"Pinterest","org_slug":"pinterest","date":"2025-07-16","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0247","shared_components":["linux-kernel"]},{"id":"IL-0138","title":{"en":"A failed Nvidia driver build left broken kernel packages on the only Kubernetes master node, which hit a kernel panic on reboot after a power outage and took the homelab cluster down for six days","ko":"nvidia-dkms-390 드라이버 빌드 실패로 망가진 커널 헤더가 몇 주 동안 방치됐다가, 정전 뒤 재부팅하자 하나뿐인 마스터 노드가 커널 패닉에 빠져 홈랩 쿠버네티스 클러스터가 6일 동안 멈췄다."},"org":"Barush Mendez","org_slug":"barush-mendez","date":"2025-03-03","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0138","shared_components":["linux-kernel"]},{"id":"IL-0216","title":{"en":"Wikimedia on-wiki search was slow or rejected after a datacenter switchover overloaded one Elasticsearch node in codfw","ko":"데이터센터 전환 직후 CirrusSearch Elasticsearch 지연이 치솟아 위키 검색이 일부 사용자에게 늦거나 안 됐다"},"org":"Wikimedia Foundation","org_slug":"wikimedia-foundation","date":"2023-09-20","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0216","shared_components":["elasticsearch-opensearch"]},{"id":"IL-0112","title":{"en":"The daily node rollover put DFDS's EKS nodes on an AMI later recalled for a kernel issue, causing \"PLEG is not healthy\" NodeNotReady flaps and pod network timeouts","ko":"나중에 회수된 EKS 노드 AMI 로 노드를 교체한 뒤 노드가 PLEG is not healthy 를 보고하며 NotReady 를 오갔고 파드 네트워크 타임아웃이 났다"},"org":"DFDS","org_slug":"dfds","date":"2022-10-31","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0112","shared_components":["linux-kernel"]},{"id":"IL-0213","title":{"en":"Roblox was down for 73 hours because key-value writes on its single Consul cluster slowed from under 300 ms to 2 seconds","ko":"Consul 클러스터의 KV 쓰기가 느려지고 리더가 뒤처지며 Roblox 전체가 73시간 동안 멈췄다"},"org":"Roblox","org_slug":"roblox","date":"2021-10-28","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0213","shared_components":["coordination-store"]},{"id":"IL-0462","title":{"en":"Some MySQL pods on Shopify's GKE clusters took up to 30 minutes to start, slowing instance replacement","ko":"GKE 위 MySQL 파드 일부가 시작과 종료에 오래 걸려 KateSQL 인스턴스 교체가 느려졌다"},"org":"Shopify","org_slug":"shopify","date":"2021-10-21","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0462","shared_components":["linux-kernel"]},{"id":"IL-0041","title":{"en":"fluent-bit silently stopped shipping application logs from one node to Elasticsearch","ko":"fluent-bit 이 오류 없이 멈춰 운영 노드 한 대의 애플리케이션 로그가 Elasticsearch 로 전혀 가지 않았다."},"org":"PrometheusKube","org_slug":"prometheuskube","date":"2020-11-11","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0041","shared_components":["elasticsearch-opensearch"]},{"id":"IL-0029","title":{"en":"Slow user-facing services from CPU throttling on containers that used far less CPU than their limit","ko":"CPU 한도를 건 컨테이너가 사용량이 한도에 한참 못 미치는데도 스로틀링되어 사용자 대면 서비스의 지연 시간이 늘었다."},"org":"Buffer","org_slug":"buffer","date":"2020-08-31","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0029","shared_components":["linux-kernel"]},{"id":"IL-0271","title":{"en":"An empty etcd service entry crashed Discord's Elixir chat nodes, and Discord was down for most users for an hour","ko":"서비스 디스커버리에 잘못된 항목이 들어가 Elixir 서비스들이 한꺼번에 죽으면서 Discord 가 대부분의 사용자에게 한 시간 동안 열리지 않았다"},"org":"Discord","org_slug":"discord","date":"2020-03-16","severity":"sev1","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0271","shared_components":["coordination-store"]},{"id":"IL-0012","title":{"en":"CPU limits and an unpatched kernel CFS quota bug throttled multi-threaded services into stalls, failed health checks and timeouts","ko":"CPU limit 과 커널 CFS 쿼터 버그로 파드가 과하게 스로틀링돼 헬스 체크가 실패하고 애플리케이션이 멈췄다"},"org":"Omio","org_slug":"omio","date":"2020-02-17","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0012","shared_components":["linux-kernel"]},{"id":"IL-0143","title":{"en":"GKE nodes upgraded to certain 1.13 and 1.14 versions had an elevated number of kernel panics, fixed by releases that downgraded the Container-Optimized OS image","ko":"GKE 1.13 과 1.14 의 특정 버전으로 업그레이드한 노드에서 커널 패닉이 늘었다."},"org":"Google Cloud","org_slug":"google-cloud","date":"2019-11-04","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0143","shared_components":["linux-kernel"]},{"id":"IL-0587","title":{"en":"Elastic Cloud proxies in AWS eu-west-1 flooded an overloaded ZooKeeper and ran out of memory, cutting cluster access","ko":"Elastic Cloud AWS eu-west-1 리전의 클러스터 접속이 약 3시간 심하게 저하되고 20분 동안 완전히 끊겼다"},"org":"Elastic","org_slug":"elastic","date":"2019-01-18","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0587","shared_components":["coordination-store"]},{"id":"IL-0439","title":{"en":"Database access slowed after each block device and filesystem expansion because the Linux kernel dropped the page cache","ko":"블록 디바이스와 파일 시스템을 확장할 때마다 데이터베이스 접근이 한동안 느려졌다"},"org":"Cybozu","org_slug":"cybozu","date":"2018-06-28","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0439","shared_components":["linux-kernel"]},{"id":"IL-0536","title":{"en":"Amazon SimpleDB in US East returned HTTP 500 to nearly all API calls for two hours after its nodes left the cluster","ko":"US East 의 Amazon SimpleDB 가 두 시간 동안 거의 모든 API 호출에 실패했다"},"org":"Amazon Web Services","org_slug":"amazon-web-services","date":"2014-06-13","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0536","shared_components":["coordination-store"]},{"id":"IL-0274","title":{"en":"GitHub's code search Elasticsearch cluster hit near-100% CPU and rapid master elections; a restart found lost shards","ko":"GitHub 코드 검색의 elasticsearch 클러스터에서 일부 노드 CPU 가 100% 에 가깝게 차고 마스터가 계속 바뀌면서 검색과 색인이 타임아웃되고 샤드 데이터가 사라졌다"},"org":"GitHub","org_slug":"github","date":"2013-01-24","severity":"unknown","root_cause":"software-bug","url":"https://benchlet.com/incidentlib/app/cases/IL-0274","shared_components":["elasticsearch-opensearch"]}]