From 8c2ff0d0878c05ac2f0c3a56f508deae7ede2d69 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 20 Aug 2022 16:29:08 +0200 Subject: [PATCH] AutoBatch protect from negative batch sizes (#9048) * AutoBatch protect from negative batch sizes Signed-off-by: Glenn Jocher * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: Glenn Jocher Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- utils/autobatch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/autobatch.py b/utils/autobatch.py index 07cddc99f400..8d12e46f0f09 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -60,6 +60,9 @@ def autobatch(model, imgsz=640, fraction=0.9, batch_size=16): i = results.index(None) # first fail index if b >= batch_sizes[i]: # y intercept above failure point b = batch_sizes[max(i - 1, 0)] # select prior safe point + if b < 1: # zero or negative batch size + b = 16 + LOGGER.warning(f'{prefix}WARNING: ⚠️ CUDA anomaly detected, recommend restart environment and retry command.') fraction = np.polyval(p, b) / t # actual fraction predicted LOGGER.info(f'{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%) ✅')