From a9dea8550ad7a55c4263df16ab047a8b8645e7e4 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 10 Jan 2021 11:42:45 -0800 Subject: [PATCH] GitHub API rate limit fix (#1894) --- utils/google_utils.py | 92 ++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/utils/google_utils.py b/utils/google_utils.py index 242270c1b033..699e28d1bcbc 100644 --- a/utils/google_utils.py +++ b/utils/google_utils.py @@ -12,71 +12,73 @@ def gsutil_getsize(url=''): # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du - s = subprocess.check_output('gsutil du %s' % url, shell=True).decode('utf-8') + s = subprocess.check_output(f'gsutil du {url}', shell=True).decode('utf-8') return eval(s.split(' ')[0]) if len(s) else 0 # bytes -def attempt_download(weights): - # Attempt to download pretrained weights if not found locally - weights = str(weights).strip().replace("'", '') - file = Path(weights).name.lower() - - msg = weights + ' missing, try downloading from https://github.com/ultralytics/yolov5/releases/' - response = requests.get('https://github.com/gitapi/repos/ultralytics/yolov5/releases/latest').json() # github api - assets = [x['name'] for x in response['assets']] # release assets, i.e. ['yolov5s.pt', 'yolov5m.pt', ...] - redundant = False # second download option - - if file in assets and not os.path.isfile(weights): - try: # GitHub - tag = response['tag_name'] # i.e. 'v1.0' - url = f'https://github.com/ultralytics/yolov5/releases/download/{tag}/{file}' - print('Downloading %s to %s...' % (url, weights)) - torch.hub.download_url_to_file(url, weights) - assert os.path.exists(weights) and os.path.getsize(weights) > 1E6 # check - except Exception as e: # GCP - print('Download error: %s' % e) - assert redundant, 'No secondary mirror' - url = 'https://storage.googleapis.com/ultralytics/yolov5/ckpt/' + file - print('Downloading %s to %s...' % (url, weights)) - r = os.system('curl -L %s -o %s' % (url, weights)) # torch.hub.download_url_to_file(url, weights) - finally: - if not (os.path.exists(weights) and os.path.getsize(weights) > 1E6): # check - os.remove(weights) if os.path.exists(weights) else None # remove partial downloads - print('ERROR: Download failure: %s' % msg) - print('') - return - - -def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m', name='tmp.zip'): +def attempt_download(file): + # Attempt file download if does not exist + file = Path(str(file).strip().replace("'", '').lower()) + + if not file.exists(): + response = requests.get('https://github.com/gitapi/repos/ultralytics/yolov5/releases/latest').json() # github api + assets = [x['name'] for x in response['assets']] # release assets, i.e. ['yolov5s.pt', 'yolov5m.pt', ...] + name = file.name + + if name in assets: + msg = f'{file} missing, try downloading from https://github.com/ultralytics/yolov5/releases/' + redundant = False # second download option + try: # GitHub + tag = response['tag_name'] # i.e. 'v1.0' + url = f'https://github.com/ultralytics/yolov5/releases/download/{tag}/{name}' + print(f'Downloading {url} to {file}...') + torch.hub.download_url_to_file(url, file) + assert file.exists() and file.stat().st_size > 1E6 # check + except Exception as e: # GCP + print(f'Download error: {e}') + assert redundant, 'No secondary mirror' + url = f'https://storage.googleapis.com/ultralytics/yolov5/ckpt/{name}' + print(f'Downloading {url} to {file}...') + os.system(f'curl -L {url} -o {file}') # torch.hub.download_url_to_file(url, weights) + finally: + if not file.exists() or file.stat().st_size < 1E6: # check + file.unlink(missing_ok=True) # remove partial downloads + print(f'ERROR: Download failure: {msg}') + return + + +def gdrive_download(id='16TiPfZj7htmTyhntwcZyEEAejOUxuT6m', file='tmp.zip'): # Downloads a file from Google Drive. from yolov5.utils.google_utils import *; gdrive_download() t = time.time() - print('Downloading https://drive.google.com/uc?export=download&id=%s as %s... ' % (id, name), end='') - os.remove(name) if os.path.exists(name) else None # remove existing - os.remove('cookie') if os.path.exists('cookie') else None + file = Path(file) + cookie = Path('cookie') # gdrive cookie + print(f'Downloading https://drive.google.com/uc?export=download&id={id} as {file}... ', end='') + file.unlink(missing_ok=True) # remove existing file + cookie.unlink(missing_ok=True) # remove existing cookie # Attempt file download out = "NUL" if platform.system() == "Windows" else "/dev/null" - os.system('curl -c ./cookie -s -L "drive.google.com/uc?export=download&id=%s" > %s ' % (id, out)) + os.system(f'curl -c ./cookie -s -L "drive.google.com/uc?export=download&id={id}" > {out}') if os.path.exists('cookie'): # large file - s = 'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm=%s&id=%s" -o %s' % (get_token(), id, name) + s = f'curl -Lb ./cookie "drive.google.com/uc?export=download&confirm={get_token()}&id={id}" -o {file}' else: # small file - s = 'curl -s -L -o %s "drive.google.com/uc?export=download&id=%s"' % (name, id) + s = f'curl -s -L -o {file} "drive.google.com/uc?export=download&id={id}"' r = os.system(s) # execute, capture return - os.remove('cookie') if os.path.exists('cookie') else None + cookie.unlink(missing_ok=True) # remove existing cookie # Error check if r != 0: - os.remove(name) if os.path.exists(name) else None # remove partial + file.unlink(missing_ok=True) # remove partial print('Download error ') # raise Exception('Download error') return r # Unzip if archive - if name.endswith('.zip'): + if file.suffix == '.zip': print('unzipping... ', end='') - os.system('unzip -q %s' % name) # unzip - os.remove(name) # remove zip to free space + os.system(f'unzip -q {file}') # unzip + file.unlink() # remove zip to free space - print('Done (%.1fs)' % (time.time() - t)) + print(f'Done ({time.time() - t:.1f}s)') return r