[{"id":"IL-0416","title":{"en":"Defective GPUs silently corrupted ByteDance's LLM training, showing as loss spikes, NaN and misleading code errors","ko":"LLM 학습에서 SDC 결함이 있는 GPU 가 ECC 오류나 온도 경보 없이 계산을 조용히 망가뜨려 손실이 치솟거나 작업이 멈췄다"},"org":"ByteDance","org_slug":"bytedance","date":"2026-07-13","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0416","shared_components":["gpu-hardware"]},{"id":"IL-0415","title":{"en":"GSP, fallen-off-the-bus and NVLink errors left A100 GPUs on NCSA's Delta inoperable and failed the jobs running on them","ko":"Delta 의 A100 GPU 에서 GSP 오류, GPU Fallen Off the Bus 오류, NVLink 오류가 나 GPU 가 오류 상태에 빠지고 그 위의 작업이 실패했다"},"org":"NCSA","org_slug":"ncsa","date":"2025-12-10","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0415","shared_components":["gpu-hardware"]},{"id":"IL-0482","title":{"en":"Bursts of uncorrectable A100 memory errors on Perlmutter and Polaris reached over 1,000 double-bit errors a day","ko":"Perlmutter와 Polaris의 A100 GPU에서 정정할 수 없는 메모리 오류가 하루 1,000건 넘게 몰려 터졌다"},"org":"NERSC","org_slug":"nersc","date":"2025-09-04","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0482","shared_components":["gpu-hardware"]},{"id":"IL-0421","title":{"en":"Errors, mostly GPU faults, crashed Alibaba LLM training jobs as NCCL errors; a job lost 31.19% of its time in June 2023","ko":"Alibaba 의 LLM 학습 클러스터에서 GPU 결함을 비롯한 오류로 작업이 거듭 죽어 2023년 6월 한 작업이 시간의 31.19% 를 잃었다"},"org":"Alibaba Group","org_slug":"alibaba-group","date":"2025-05-23","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0421","shared_components":["gpu-hardware"]},{"id":"IL-0411","title":{"en":"Lemon nodes in Meta's RSC-1 and RSC-2 GPU clusters failed job after job without being caught by health checks","ko":"헬스 체크를 통과하는 레몬 노드가 Meta 연구 클러스터의 학습 작업을 거듭 실패시켰다"},"org":"Meta","org_slug":"meta","date":"2024-10-29","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0411","shared_components":["gpu-hardware"]},{"id":"IL-0238","title":{"en":"Xid 74 NVLink errors led the GPU faults and InfiniBand flash cuts failed tasks on DeepSeek's 10,000-GPU A100 cluster","ko":"PCIe A100 1만 장 규모의 Fire-Flyer 2 학습 클러스터에서 GPU Xid 오류와 InfiniBand 링크의 순간 끊김으로 학습 작업이 끊기고 실패했다"},"org":"DeepSeek","org_slug":"deepseek","date":"2024-08-31","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0238","shared_components":["gpu-hardware"]},{"id":"IL-0127","title":{"en":"Llama 3 405B pre-training on 16K H100 GPUs was interrupted 419 times unexpectedly in 54 days, mostly by faulty GPUs, HBM3 memory and other hardware","ko":"Llama 3 405B 사전 학습 54일 동안 학습 작업이 예기치 않게 419번 중단되었고, 주된 원인은 GPU 와 HBM3 메모리 고장이었다"},"org":"Meta","org_slug":"meta","date":"2024-07-23","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0127","shared_components":["gpu-hardware"]},{"id":"IL-0239","title":{"en":"LLM pretraining jobs on Shanghai AI Laboratory's A100 clusters kept failing midway, mostly from NVLink and GPU errors","ko":"LLM 개발 클러스터 Acme 의 Seren 과 Kalos 에서 사전 학습 작업이 하드웨어, 네트워크, 프레임워크 오류로 자주 실패해 학습 진행을 잃었다"},"org":"Shanghai AI Laboratory","org_slug":"shanghai-ai-laboratory","date":"2024-04-03","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0239","shared_components":["gpu-hardware"]},{"id":"IL-0429","title":{"en":"GLM-130B training on 96 nodes kept stopping on node failures, and a Lustre error left the 2.3T corpus unreadable","ko":"GLM-130B 사전 학습이 노드 고장으로 거듭 중단됐고 Lustre 오류로 2.3T 말뭉치를 읽지 못했다"},"org":"Tsinghua University","org_slug":"tsinghua-university","date":"2022-05-07","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0429","shared_components":["gpu-hardware"]},{"id":"IL-0427","title":{"en":"A GPU with uncorrectable ECC errors crashed tr1-13B training, and Slurm's next job went to the same node and crashed too","ko":"tr1-13B 학습이 고장 난 GPU 가 있는 노드에서 연달아 두 번 죽었다"},"org":"BigScience","org_slug":"bigscience","date":"2021-08-12","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0427","shared_components":["gpu-hardware"]}]