From 378742e1b60e94d12884b3e61f834a50e77689cb Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 16 Sep 2022 21:17:26 +0200 Subject: [PATCH 1/4] AutoBatch `cudnn.benchmark=True` fix May resolve https://github.com/ultralytics/yolov5/issues/9287 Signed-off-by: Glenn Jocher --- utils/autobatch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/autobatch.py b/utils/autobatch.py index 641b055b9fe3..bafff1c20658 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -7,6 +7,7 @@ import numpy as np import torch +from torch.backends import cudnn from utils.general import LOGGER, colorstr from utils.torch_utils import profile @@ -46,11 +47,14 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): # Profile batch sizes batch_sizes = [1, 2, 4, 8, 16] + bm = cudnn.benchmark + cudnn.benchmark = False # avoid benchmark interference https://github.com/ultralytics/yolov5/issues/9287 try: img = [torch.empty(b, 3, imgsz, imgsz) for b in batch_sizes] results = profile(img, model, n=3, device=device) except Exception as e: LOGGER.warning(f'{prefix}{e}') + cudnn.benchmark = bm # reset to original value # Fit a solution y = [x[2] for x in results if x] # memory [2] From e9c4f8deffbd6a43b2cb840edd49846b01b6ad66 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 16 Sep 2022 21:25:41 +0200 Subject: [PATCH 2/4] Update autobatch.py Signed-off-by: Glenn Jocher --- utils/autobatch.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index bafff1c20658..9f993abdcad8 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -7,7 +7,6 @@ import numpy as np import torch -from torch.backends import cudnn from utils.general import LOGGER, colorstr from utils.torch_utils import profile @@ -34,6 +33,9 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): if device.type == 'cpu': LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}') return batch_size + if torch.backends.cudnn.benchmark: + LOGGER.info(f'{prefix}requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}') + return batch_size # Inspect CUDA memory gb = 1 << 30 # bytes to GiB (1024 ** 3) @@ -47,14 +49,11 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): # Profile batch sizes batch_sizes = [1, 2, 4, 8, 16] - bm = cudnn.benchmark - cudnn.benchmark = False # avoid benchmark interference https://github.com/ultralytics/yolov5/issues/9287 try: img = [torch.empty(b, 3, imgsz, imgsz) for b in batch_sizes] results = profile(img, model, n=3, device=device) except Exception as e: LOGGER.warning(f'{prefix}{e}') - cudnn.benchmark = bm # reset to original value # Fit a solution y = [x[2] for x in results if x] # memory [2] From 03c659d3dddfec2935431b853f9d6e80572fd611 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 16 Sep 2022 21:28:19 +0200 Subject: [PATCH 3/4] Update autobatch.py Signed-off-by: Glenn Jocher --- utils/autobatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index 9f993abdcad8..3204fd26fc41 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -34,7 +34,7 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}') return batch_size if torch.backends.cudnn.benchmark: - LOGGER.info(f'{prefix}requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}') + LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}') return batch_size # Inspect CUDA memory From 9316e342f793901f4928f0417a2153787b9600ed Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 16 Sep 2022 21:32:07 +0200 Subject: [PATCH 4/4] Update general.py Signed-off-by: Glenn Jocher --- utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/general.py b/utils/general.py index 8633511f89f5..af95b3dc2b8b 100644 --- a/utils/general.py +++ b/utils/general.py @@ -223,7 +223,7 @@ def init_seeds(seed=0, deterministic=False): torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe - torch.backends.cudnn.benchmark = True # for faster training + # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287 if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213 torch.use_deterministic_algorithms(True) torch.backends.cudnn.deterministic = True