RF/ENH: download - reimplementation to support original and upcoming UI/API

dandi/cli/cmd_download.py

@@ -25,6 +25,13 @@
     default="refresh",
     show_default=True,
 )
+@click.option(
+    "-f",
+    "--format",
+    help="Choose the format/frontend for output. TODO: support all of the ls",
+    type=click.Choice(["pyout", "debug"]),
+    default="pyout",
+)
 @click.option(
     "-J",
     "--jobs",
@@ -48,12 +55,16 @@
 )
 @click.argument("url", nargs=-1)
 @map_to_click_exceptions
-def download(url, output_dir, existing, jobs=6, develop_debug=False):
+def download(url, output_dir, existing, jobs=6, format="pytout"):
     """Download a file or entire folder from DANDI"""
     # First boring attempt at click commands being merely an interface to
     # Python function
     from ..download import download
 
     return download(
-        url, output_dir, existing=existing, jobs=jobs, develop_debug=develop_debug
+        url,
+        output_dir,
+        existing=existing,
+        format=format,
+        jobs=jobs,  # develop_debug=develop_debug
     )

dandi/cli/cmd_ls.py

@@ -82,7 +82,7 @@ def ls(paths, fields=None, format="auto", recursive=False):
         if fields and fields[0] != "path":
             # we must always have path - our "id"
             fields = ["path"] + fields
-        out = PYOUTFormatter(files=files, fields=fields)
+        out = PYOUTFormatter(fields=fields)
     elif format == "json":
         out = JSONFormatter()
     elif format == "json_pp":

dandi/cli/formatter.py

@@ -55,8 +55,7 @@ def __call__(self, rec):
 
 
 class PYOUTFormatter(pyout.Tabular):
-    def __init__(self, files, fields):
-        # max_filename_len = max(map(lambda x: len(op.basename(x)), files))
+    def __init__(self, fields):
         PYOUT_STYLE = pyouts.get_style(hide_if_missing=not fields)
 
         kw = dict(style=PYOUT_STYLE)

dandi/consts.py

@@ -4,6 +4,8 @@
 # A list of metadata fields which dandi extracts from .nwb files.
 # Additional fields (such as `number_of_*`) might be added by the
 # get_metadata`
+import os
+
 metadata_nwb_file_fields = (
     "experiment_description",
     "experimenter",
@@ -73,23 +75,31 @@
 dandiset_metadata_file = "dandiset.yaml"
 dandiset_identifier_regex = "^[0-9]{6}$"
 
-dandi_instance = namedtuple("dandi_instance", ("girder", "gui", "redirector"))
+dandi_instance = namedtuple("dandi_instance", ("girder", "gui", "redirector", "api"))
 
 known_instances = {
     "local-girder-only": dandi_instance(
-        "http://localhost:8080", None, None
+        "http://localhost:8080", None, None, None
     ),  # just pure girder
     # Redirector: TODO https://github.com/dandi/dandiarchive/issues/139
     "local-docker": dandi_instance(
-        "http://localhost:8080", "http://localhost:8085", None
+        "http://localhost:8080",
+        "http://localhost:8085",
+        None,
+        "http://localhost:9000",  # ATM it is minio, not sure where /api etc
+        # may be https://github.com/dandi/dandi-publish/pull/71 would help
     ),
     "local-docker-tests": dandi_instance(
-        "http://localhost:8081", "http://localhost:8086", "http://localhost:8079"
+        "http://localhost:8081",
+        "http://localhost:8086",
+        "http://localhost:8079",
+        None,  # TODO: https://github.com/dandi/dandi-cli/issues/164
     ),
     "dandi": dandi_instance(
         "https://girder.dandiarchive.org",
         "https://gui.dandiarchive.org",
         "https://dandiarchive.org",
+        "https://publish.dandiarchive.org/api",  # ? might become api.
     ),
 }
 # to map back url: name
@@ -100,6 +110,14 @@
 
 file_operation_modes = ["dry", "simulate", "copy", "move", "hardlink", "symlink"]
 
+#
+# Download (upload?) specific constants
+#
+# Chunk size when iterating a download (and upload) body. Taken from girder-cli
+# TODO: should we make them smaller for download than for upload?
+# ATM used only in download
+MAX_CHUNK_SIZE = os.environ.get("DANDI_MAX_CHUNK_SIZE", 1024 * 1024 * 8)  # 64
+
 #
 # Some routes
 # TODO: possibly centralize in dandi-common from our redirection service