From f5957fac0d0e782affd88eb4e8746134f7fc4e1b Mon Sep 17 00:00:00 2001
From: "John T. Wodder II" <git@varonathe.org>
Date: Mon, 11 Oct 2021 12:19:32 -0400
Subject: [PATCH] Retry on more statuses

---
 dandi/consts.py       | 4 ++++
 dandi/dandiapi.py     | 5 +++--
 dandi/dandiarchive.py | 4 ++--
 dandi/download.py     | 4 ++--
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/dandi/consts.py b/dandi/consts.py
index c7da8b6c0..0ce106b8e 100644
--- a/dandi/consts.py
+++ b/dandi/consts.py
@@ -119,3 +119,7 @@
 
 #: The identifier for draft Dandiset versions
 DRAFT = "draft"
+
+#: HTTP response status codes that should always be retried (until we run out
+#: of retries)
+RETRY_STATUSES = (500, 502, 503, 504)
diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py
index 6da4bef4c..abe43e219 100644
--- a/dandi/dandiapi.py
+++ b/dandi/dandiapi.py
@@ -69,6 +69,7 @@
 from .consts import (
     DRAFT,
     MAX_CHUNK_SIZE,
+    RETRY_STATUSES,
     DandiInstance,
     known_instances,
     known_instances_rev,
@@ -151,7 +152,7 @@ def request(
 
         :type json_resp: bool
         :param retry_statuses: a sequence of HTTP response status codes to
-            retry; 503 will be added to this set
+            retry in addition to `dandi.consts.RETRY_STATUSES`
         """
 
         url = self.get_url(path)
@@ -191,7 +192,7 @@ def request(
                         headers=headers,
                         **kwargs,
                     )
-                    if result.status_code in [503, *retry_statuses]:
+                    if result.status_code in [*RETRY_STATUSES, *retry_statuses]:
                         result.raise_for_status()
         except Exception:
             lgr.exception("HTTP connection failed")
diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py
index 692b1cfd9..468163dbc 100644
--- a/dandi/dandiarchive.py
+++ b/dandi/dandiarchive.py
@@ -9,7 +9,7 @@
 import requests
 
 from . import get_logger
-from .consts import VERSION_REGEX, known_instances
+from .consts import RETRY_STATUSES, VERSION_REGEX, known_instances
 from .dandiapi import BaseRemoteAsset, DandiAPIClient, RemoteDandiset
 from .exceptions import FailedToConnectError, NotFoundError, UnknownURLError
 from .utils import get_instance
@@ -526,7 +526,7 @@ def follow_redirect(url):
         i = 0
         while True:
             r = requests.head(url, allow_redirects=True)
-            if r.status_code in (400, 502, 503, 504) and i < 5:
+            if r.status_code in (400, *RETRY_STATUSES) and i < 5:
                 sleep(0.1 * 10 ** i)
                 i += 1
                 continue
diff --git a/dandi/download.py b/dandi/download.py
index 2797d77a7..b172d4169 100644
--- a/dandi/download.py
+++ b/dandi/download.py
@@ -12,7 +12,7 @@
 import requests
 
 from . import get_logger
-from .consts import dandiset_metadata_file
+from .consts import RETRY_STATUSES, dandiset_metadata_file
 from .dandiarchive import DandisetURL, MultiAssetURL, SingleAssetURL, parse_dandi_url
 from .dandiset import Dandiset
 from .support.digests import get_digest
@@ -566,7 +566,7 @@ def _download_file(
             if attempt >= 2 or exc.response.status_code not in (
                 400,  # Bad Request, but happened with gider:
                 # https://github.com/dandi/dandi-cli/issues/87
-                503,  # Service Unavailable
+                *RETRY_STATUSES,
             ):
                 lgr.debug("Download failed: %s", exc)
                 yield {"status": "error", "message": str(exc)}