Skip to content

Commit

Permalink
Fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
jwodder committed Aug 25, 2020
1 parent 39979d7 commit eb7e0d5
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 10 deletions.
23 changes: 16 additions & 7 deletions dandi/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import random
import re
import requests
import socket
import sys
import time

Expand Down Expand Up @@ -191,7 +192,7 @@ def parse(cls, url, *, map_instance=True):
url_server = groups["server"]
server = cls.map_to[server_type].get(url_server.rstrip("/"), url_server)

if not server.endswith("/"):
if server and not server.endswith("/"):
server += "/" # we expected '/' to be there so let it be

if server_type == "girder":
Expand Down Expand Up @@ -581,7 +582,11 @@ def _map_to_girder(url):
# This is a duplicate call from above but it is cheap, so decided to just redo
# it here instead of passing all the variables + url
server_type, server_url, asset_type, asset_id = parse_dandi_url(url)
server_url = known_instances[known_instances_rev[server_url.rstrip("/")]].girder
server_url = known_instances[
known_instances_rev[server_url.rstrip("/")]
if server_url
else "local-docker-tests"
].girder
server_type = "girder"
client = girder.get_client(server_url, authenticate=False, progressbars=True)
# TODO: RF if https://github.com/dandi/dandiarchive/issues/316 gets addressed
Expand Down Expand Up @@ -768,13 +773,17 @@ def _download_file(
else:
break
# both girder and we use HttpError
except requests.exceptions.HTTPError as exc:
except (requests.exceptions.RequestException, socket.timeout) as exc:
# TODO: actually we should probably retry only on selected codes, and also
# respect Retry-After
if attempt >= 2 or exc.response.status_code not in (
400, # Bad Request, but happened with gider:
# https://github.com/dandi/dandi-cli/issues/87
503, # Service Unavailable
if attempt >= 2 or (
isinstance(exc, requests.exceptions.HTTPError)
and exc.response.status_code
not in (
400, # Bad Request, but happened with gider:
# https://github.com/dandi/dandi-cli/issues/87
503, # Service Unavailable
)
):
lgr.debug("Download failed: %s", exc)
yield {"status": "error", "message": str(exc)}
Expand Down
6 changes: 3 additions & 3 deletions dandi/girder.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,11 +375,11 @@ def downloader(start_at=0):
for attempt in range(3):
try:
path = f"file/{file_id}/download"
headers = None
params = None
if start_at > 0:
headers = {"Range": f"bytes={start_at}-"}
params = {"offset": str(start_at)}
resp = self.sendRestRequest(
"get", path, stream=True, jsonResp=False, headers=headers
"get", path, stream=True, jsonResp=False, parameters=params
)
return resp.iter_content(chunk_size=chunk_size)
except gcl.HttpError as exc:
Expand Down

0 comments on commit eb7e0d5

Please sign in to comment.