[{"id":"IL-0177","title":{"en":"WebSocket connections to LINE's messaging-hub on Kubernetes dropped all at once whenever another service's change reloaded a shared HAProxy L7 load balancer, and the reconnect wave overloaded a token API and filled nf_conntrack on Nginx","ko":"다른 팀의 설정 배포로 공유 L7 HAProxy 가 재로드될 때마다 messaging-hub 의 웹소켓 연결이 한꺼번에 끊겼다."},"org":"LINE+","org_slug":"line","date":"2022-10-14","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0177","shared_components":["cloud-load-balancer"]},{"id":"IL-0191","title":{"en":"A label selector accidentally added to the Loki distributor Service matched only the new pods of a rollout, which took all write traffic, were OOM-killed and stalled the rollout","ko":"Loki distributor 서비스에 레이블 셀렉터가 하나 더 들어가 롤아웃 중 새 파드 몇 개에만 쓰기 트래픽이 몰렸고, 그 파드들이 OOM으로 죽으면서 로그 쓰기 경로가 약 25분간 멈췄다."},"org":"Grafana Labs","org_slug":"grafana-labs","date":"2022-09-01","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0191","shared_components":["config-templating"]},{"id":"IL-0009","title":{"en":"A duplicated generated name made Argo CD replace the InfluxDB Cloud production Application and delete the whole production environment","ko":"ArgoCD Application 이름이 겹쳐 InfluxDB Cloud 운영 환경 하나가 통째로 삭제됐다"},"org":"InfluxData","org_slug":"influxdata","date":"2022-06-24","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0009","shared_components":["config-templating"]},{"id":"IL-0208","title":{"en":"A GitLab chart bump that renamed Kubernetes Service and Deployment ports made the GCP load balancer reconfigure, and GitLab.com slowed down and returned intermittent 50x errors","ko":"GitLab Helm 차트 업그레이드가 Service 와 Deployment 의 포트 이름을 바꿔 GCP 로드 밸런서가 다시 구성되는 동안 GitLab.com 요청이 느려지고 50x 오류가 났다"},"org":"GitLab","org_slug":"gitlab","date":"2022-03-31","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0208","shared_components":["cloud-load-balancer"]},{"id":"IL-0111","title":{"en":"Switching the main Kubernetes ingress controller from Traefik v1 to v2 stopped X-Forwarded headers being passed, which broke the Okta login flow for every service using Okta and stopped DFDS online bookings for over an hour","ko":"메인 인그레스 컨트롤러를 Traefik v2 로 바꾸자 X-Forwarded 헤더가 전달되지 않아 Okta 로그인이 깨졌고, DFDS 온라인 예약이 한 시간 넘게 멈췄다"},"org":"DFDS","org_slug":"dfds","date":"2021-08-26","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0111","shared_components":["cloud-load-balancer"]},{"id":"IL-0010","title":{"en":"A templating change that dropped its {{ }} delimiters made Argo CD delete all 478 Skyscanner services in every region","ko":"템플릿 중괄호가 빠진 설정 변경으로 ArgoCD 가 전 세계 모든 클러스터의 서비스를 지웠다"},"org":"Skyscanner","org_slug":"skyscanner","date":"2021-08-25","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0010","shared_components":["config-templating"]},{"id":"IL-0105","title":{"en":"Replacing the AWS VPC CNI during a Terraform refactoring detached the ALB target groups, and every service in the Kubernetes cluster returned 503","ko":"VPC CNI 교체로 ALB 에서 타깃 그룹이 떨어져 나가 쿠버네티스 클러스터의 모든 서비스가 503 을 냈다"},"org":"DFDS","org_slug":"dfds","date":"2021-06-22","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0105","shared_components":["cloud-load-balancer"]},{"id":"IL-0052","title":{"en":"A cryptocurrency miner ran on every node of the development and staging clusters through a public, unauthenticated Weave Scope dashboard","ko":"인증 없이 인터넷에 열린 Weave Scope 대시보드를 통해 암호화폐 채굴기가 개발 클러스터와 스테이징 클러스터의 모든 노드에서 돌았다."},"org":"JW Player","org_slug":"jw-player","date":"2019-03-19","severity":"unknown","root_cause":"config-change","url":"https://benchlet.com/incidentlib/app/cases/IL-0052","shared_components":["cloud-load-balancer"]}]