diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py index 82a398c4..cfdf80ed 100644 --- a/ultralytics/engine/trainer.py +++ b/ultralytics/engine/trainer.py @@ -225,9 +225,9 @@ class BaseTrainer: torch.cuda.set_device(RANK) self.device = torch.device("cuda", RANK) # LOGGER.info(f'DDP info: RANK {RANK}, WORLD_SIZE {world_size}, DEVICE {self.device}') - os.environ["TORCH_NCCL_BLOCKING_WAIT"] = "1" # set to enforce timeout + os.environ["NCCL_BLOCKING_WAIT"] = "1" # set to enforce timeout dist.init_process_group( - backend="nccl" if dist.is_nccl_available() else "gloo", + "nccl" if dist.is_nccl_available() else "gloo", timeout=timedelta(seconds=10800), # 3 hours rank=RANK, world_size=world_size,