try nccl

train.py

@@ -533,7 +533,7 @@ def train(hyp,  # path/to/hyp.yaml or hyp dictionary
         assert torch.cuda.device_count() > LOCAL_RANK, 'too few GPUS for DDP command'
         torch.cuda.set_device(LOCAL_RANK)
         device = torch.device('cuda', LOCAL_RANK)
-        dist.init_process_group(backend="gloo")  # distributed backend
+        dist.init_process_group(backend="nccl")  # distributed backend
         assert opt.batch_size % WORLD_SIZE == 0, '--batch-size must be multiple of CUDA device count'
         assert not opt.image_weights, '--image-weights argument is not compatible with DDP training'
         opt.batch_size = opt.total_batch_size // WORLD_SIZE