Skip to content

Commit

Permalink
check batch_size % utilized_device_count (ultralytics#3276)
Browse files Browse the repository at this point in the history
Bug fix to check batch_size divisibility of utilized CUDA device count vs total system CUDA device count.
  • Loading branch information
glenn-jocher committed May 21, 2021
1 parent 10d56d7 commit f340235
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions utils/torch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,12 @@ def select_device(device='', batch_size=None):

cuda = not cpu and torch.cuda.is_available()
if cuda:
n = torch.cuda.device_count()
if n > 1 and batch_size: # check that batch_size is compatible with device_count
devices = device.split(',') if device else range(torch.cuda.device_count()) # i.e. 0,1,6,7
n = len(devices) # device count
if n > 1 and batch_size: # check batch_size is divisible by device_count
assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}'
space = ' ' * len(s)
for i, d in enumerate(device.split(',') if device else range(n)):
for i, d in enumerate(devices):
p = torch.cuda.get_device_properties(i)
s += f"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / 1024 ** 2}MB)\n" # bytes to MB
else:
Expand Down

0 comments on commit f340235

Please sign in to comment.