[{"id":"IL-0586","title":{"en":"Four NIC-to-T0 links flapped in rapid succession and cut a 50K-GPU pretraining job's throughput about 25% for a minute","ko":"T0 스위치 트랜시버의 링크 네 개가 연달아 플래핑해 50K GPU 사전 학습 작업의 처리량이 1분 동안 약 25% 떨어졌다"},"org":"OpenAI","org_slug":"openai","date":"2026-05-05","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0586","shared_components":["network-link"]},{"id":"IL-0421","title":{"en":"Errors, mostly GPU faults, crashed Alibaba LLM training jobs as NCCL errors; a job lost 31.19% of its time in June 2023","ko":"Alibaba 의 LLM 학습 클러스터에서 GPU 결함을 비롯한 오류로 작업이 거듭 죽어 2023년 6월 한 작업이 시간의 31.19% 를 잃었다"},"org":"Alibaba Group","org_slug":"alibaba-group","date":"2025-05-23","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0421","shared_components":["nccl"]},{"id":"IL-0127","title":{"en":"Llama 3 405B pre-training on 16K H100 GPUs was interrupted 419 times unexpectedly in 54 days, mostly by faulty GPUs, HBM3 memory and other hardware","ko":"Llama 3 405B 사전 학습 54일 동안 학습 작업이 예기치 않게 419번 중단되었고, 주된 원인은 GPU 와 HBM3 메모리 고장이었다"},"org":"Meta","org_slug":"meta","date":"2024-07-23","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0127","shared_components":["nccl"]},{"id":"IL-0418","title":{"en":"Machine, ICI link and optical circuit switch failures interrupted training jobs on Google's TPU v4 supercomputers","ko":"TPU v4 슈퍼컴퓨터에서 장비, ICI 링크, 광 회로 스위치가 고장 나 학습 작업 약 1% 가 멈추거나 느려졌다"},"org":"Google","org_slug":"google","date":"2024-04-16","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0418","shared_components":["network-link"]},{"id":"IL-0239","title":{"en":"LLM pretraining jobs on Shanghai AI Laboratory's A100 clusters kept failing midway, mostly from NVLink and GPU errors","ko":"LLM 개발 클러스터 Acme 의 Seren 과 Kalos 에서 사전 학습 작업이 하드웨어, 네트워크, 프레임워크 오류로 자주 실패해 학습 진행을 잃었다"},"org":"Shanghai AI Laboratory","org_slug":"shanghai-ai-laboratory","date":"2024-04-03","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0239","shared_components":["nccl"]},{"id":"IL-0540","title":{"en":"Korea's local government administrative services and Government24 went down as a router port dropped most large packets","ko":"센터 사이를 잇는 라우터의 포트에서 큰 패킷이 유실돼 지방행정전산서비스와 정부24가 하루 넘게 멈췄다"},"org":"Korea Ministry of the Interior and Safety","org_slug":"korea-ministry-of-the-interior-and-safety","date":"2023-11-17","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0540","shared_components":["network-link"]},{"id":"IL-0125","title":{"en":"A GPU died about once a week during a 176B-parameter training run, and sometimes the job hung instead of exiting, so SLURM never restarted it","ko":"Jean Zay 에서 진행한 176B 다국어 모델 학습에서 GPU 가 고장 나도 학습 작업이 끝나지 않고 멈춰 버려 SLURM 이 작업을 자동으로 재시작하지 못했다."},"org":"BigScience","org_slug":"bigscience","date":"2022-05-18","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0125","shared_components":["nccl"]}]