AutoBatch cudnn.benchmark=True fix (#9448)

* AutoBatch `cudnn.benchmark=True` fix May resolve #9287 Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> * Update autobatch.py Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> * Update autobatch.py Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> * Update general.py Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com> Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
ultralytics · Sep 16, 2022 · 5e1a955 · 5e1a955
1 parent db06f49
commit 5e1a955
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/utils/autobatch.py b/utils/autobatch.py
@@ -33,6 +33,9 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16):
     if device.type == 'cpu':
         LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}')
         return batch_size
+    if torch.backends.cudnn.benchmark:
+        LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}')
+        return batch_size
 
     # Inspect CUDA memory
     gb = 1 << 30  # bytes to GiB (1024 ** 3)

diff --git a/utils/general.py b/utils/general.py
@@ -223,7 +223,7 @@ def init_seeds(seed=0, deterministic=False):
     torch.manual_seed(seed)
     torch.cuda.manual_seed(seed)
     torch.cuda.manual_seed_all(seed)  # for Multi-GPU, exception safe
-    torch.backends.cudnn.benchmark = True  # for faster training
+    # torch.backends.cudnn.benchmark = True  # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287
     if deterministic and check_version(torch.__version__, '1.12.0'):  # https://github.com/ultralytics/yolov5/pull/8213
         torch.use_deterministic_algorithms(True)
         torch.backends.cudnn.deterministic = True