[{"id":"IL-0127","title":{"en":"Llama 3 405B pre-training on 16K H100 GPUs was interrupted 419 times unexpectedly in 54 days, mostly by faulty GPUs, HBM3 memory and other hardware","ko":"Llama 3 405B 사전 학습 54일 동안 학습 작업이 예기치 않게 419번 중단되었고, 주된 원인은 GPU 와 HBM3 메모리 고장이었다"},"org":"Meta","org_slug":"meta","date":"2024-07-23","severity":"unknown","root_cause":"hardware-failure","url":"https://benchlet.com/incidentlib/app/cases/IL-0127","shared_components":["nccl","pytorch"]}]