From 818bacf88d423c057390be6622e4022e04ce9253 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 20 Aug 2022 16:13:18 +0200 Subject: [PATCH 1/4] AutoBatch protect from negative batch sizes Signed-off-by: Glenn Jocher --- utils/autobatch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/autobatch.py b/utils/autobatch.py index 07cddc99f400..56ffcdb6ac4f 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -60,6 +60,9 @@ def autobatch(model, imgsz=640, fraction=0.9, batch_size=16): i = results.index(None) # first fail index if b >= batch_sizes[i]: # y intercept above failure point b = batch_sizes[max(i - 1, 0)] # select prior safe point + if b < 1: # zero or negative batch size + b = 16 + LOGGER.warning(f'{prefix}WARNING: ⚠️ CUDA anomaly detected, recommend restart environment and retry command.') fraction = np.polyval(p, b) / t # actual fraction predicted LOGGER.info(f'{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%) ✅') From dced7c0231a445e9b8505068d68be746838af4ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Aug 2022 14:13:40 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- utils/autobatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index 56ffcdb6ac4f..8d12e46f0f09 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -60,7 +60,7 @@ def autobatch(model, imgsz=640, fraction=0.9, batch_size=16): i = results.index(None) # first fail index if b >= batch_sizes[i]: # y intercept above failure point b = batch_sizes[max(i - 1, 0)] # select prior safe point - if b < 1: # zero or negative batch size + if b < 1: # zero or negative batch size b = 16 LOGGER.warning(f'{prefix}WARNING: ⚠️ CUDA anomaly detected, recommend restart environment and retry command.') From 7d2e3a76047e352245711d045afc4c11caf853e5 Mon Sep 17 00:00:00 2001 From: glennjocher Date: Sat, 20 Aug 2022 16:14:43 +0200 Subject: [PATCH 3/4] cleanup --- utils/autobatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index 56ffcdb6ac4f..1ceb59014b5b 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -61,7 +61,7 @@ def autobatch(model, imgsz=640, fraction=0.9, batch_size=16): if b >= batch_sizes[i]: # y intercept above failure point b = batch_sizes[max(i - 1, 0)] # select prior safe point if b < 1: # zero or negative batch size - b = 16 + b = batch_size # set to default LOGGER.warning(f'{prefix}WARNING: ⚠️ CUDA anomaly detected, recommend restart environment and retry command.') fraction = np.polyval(p, b) / t # actual fraction predicted From d978477fc5aa37d50bca46211595fadb4e9ec927 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Aug 2022 14:15:30 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- utils/autobatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/autobatch.py b/utils/autobatch.py index 56ffcdb6ac4f..8d12e46f0f09 100644 --- a/utils/autobatch.py +++ b/utils/autobatch.py @@ -60,7 +60,7 @@ def autobatch(model, imgsz=640, fraction=0.9, batch_size=16): i = results.index(None) # first fail index if b >= batch_sizes[i]: # y intercept above failure point b = batch_sizes[max(i - 1, 0)] # select prior safe point - if b < 1: # zero or negative batch size + if b < 1: # zero or negative batch size b = 16 LOGGER.warning(f'{prefix}WARNING: ⚠️ CUDA anomaly detected, recommend restart environment and retry command.')