Improve CLI documentation

dandi/cli/cmd_organize.py

@@ -9,9 +9,9 @@
 
 @click.command()
 @dandiset_path_option(
-    help="A top directory (local) of the dandiset to organize files under. "
-    "If not specified, dandiset current directory is under is assumed. "
-    "For 'simulate' mode target dandiset/directory must not exist.",
+    help="The root directory of the Dandiset to organize files under. "
+    "If not specified, the Dandiset under the current directory is assumed. "
+    "For 'simulate' mode, the target Dandiset/directory must not exist.",
     type=click.Path(dir_okay=True, file_okay=False),
 )
 @click.option(
@@ -25,8 +25,8 @@
     "-f",
     "--files-mode",
     help="If 'dry' - no action is performed, suggested renames are printed. "
-    "If 'simulate' - hierarchy of empty files at --local-top-path is created. "
-    "Note that previous layout should be removed prior this operation.  "
+    "If 'simulate' - hierarchy of empty files at --dandiset-path is created. "
+    "Note that previous layout should be removed prior to this operation.  "
     "If 'auto' - whichever of symlink, hardlink, copy is allowed by system. "
     "The other modes (copy, move, symlink, hardlink) define how data files "
     "should be made available.",
@@ -47,30 +47,32 @@ def organize(
     """(Re)organize files according to the metadata.
 
     The purpose of this command is to take advantage of metadata contained in
-    the .nwb files to provide datasets with consistently named files, so their
-    naming reflects data they contain.
+    .nwb files to provide datasets with consistently-named files whose names
+    reflect the data they contain.
 
-    .nwb files are organized into a hierarchy of subfolders one per each
-    "subject", e.g. sub-0001 if .nwb file had contained a Subject group with
-    subject_id=0001.  Each file in a subject-specific subfolder follows the
-    convention:
+    .nwb files are organized into a hierarchy of subfolders, one per "subject",
+    e.g., `sub-0001` if an .nwb file contained a Subject group with
+    `subject_id=0001`.  Each file in a subject-specific subfolder follows the
+    pattern:
 
         sub-<subject_id>[_key-<value>][_mod1+mod2+...].nwb
 
-    where following keys are considered if present in the data:
+    where the following keys are considered if present in the data:
 
-        ses -- session_id\n
-        tis -- tissue_sample_id\n
-        slice -- slice_id\n
-        cell -- cell_id\n
+    \b
+        ses -- session_id
+        tis -- tissue_sample_id
+        slice -- slice_id
+        cell -- cell_id
 
-    and `modX` are "modalities" as identified based on detected neural data types
-    (such as "ecephys", "icephys") per extensions found in nwb-schema definitions:
+    and `modX` are "modalities" as identified based on detected neural data
+    types (such as "ecephys", "icephys") per extensions found in nwb-schema
+    definitions:
     https://github.com/NeurodataWithoutBorders/nwb-schema/tree/dev/core
 
-    In addition an "obj" key with a value corresponding to crc32 checksum of
-    "object_id" is added if aforementioned keys and the list of modalities are
-    not sufficient to disambiguate different files.
+    In addition, an "obj" key with a value corresponding to the crc32 checksum
+    of "object_id" is added if the aforementioned keys and the list of
+    modalities are not sufficient to disambiguate different files.
 
     You can visit https://dandiarchive.org for a growing collection of
     (re)organized dandisets.
@@ -118,7 +120,7 @@ def act(func, *args, **kwargs):
         if op.exists(dandiset_path):
             # TODO: RF away
             raise RuntimeError(
-                "In simulate mode %r (--top-path) must not exist, we will create it."
+                "In simulate mode %r (--dandiset-path) must not exist, we will create it."
                 % dandiset_path
             )
 

dandi/cli/cmd_upload.py

@@ -93,17 +93,18 @@ def upload(
     upload_dandiset_metadata=False,
     devel_debug=False,
 ):
-    """Upload dandiset (files) to DANDI archive.
+    """
+    Upload Dandiset files to DANDI Archive.
 
-    Target dandiset to upload to must already be registered in the archive and
-    locally "dandiset.yaml" should exist in `--dandiset-path`.
+    The target Dandiset to upload to must already be registered in the archive,
+    and a `dandiset.yaml` file must exist in the local `--dandiset-path`.
 
-    Local dandiset should pass validation.  For that it should be first organized
-    using 'dandiset organize' command.
+    Local Dandiset should pass validation.  For that, the assets should first
+    be organized using the `dandi organize` command.
 
-    By default all files in the dandiset (not following directories starting with a period)
-    will be considered for the upload.  You can point to specific files you would like to
-    validate and have uploaded.
+    By default all .nwb files in the Dandiset (excluding directories starting
+    with a period) will be considered for the upload.  You can point to
+    specific files you would like to validate and have uploaded.
     """
     from ..upload import upload
 

dandi/dandiarchive.py

@@ -1,7 +1,8 @@
 """
-This module provides functionality for parsing URLs for Dandisets & assets on
-Dandi Archive servers and for fetching the objects to which the URLs refer.
-See the docstring for `parse_dandi_url()` for a list of accepted URL formats.
+This module provides functionality for parsing URLs and other resource
+identifiers for Dandisets & assets on Dandi Archive servers and for fetching
+the objects to which the URLs refer.  See :ref:`resource_ids` for a list of
+accepted URL formats.
 
 Basic operation begins by calling `parse_dandi_url()` on a URL in order to
 acquire a `ParsedDandiURL` instance, which can then be used to obtain the
@@ -479,57 +480,8 @@ class _dandi_url_parser:
     @classmethod
     def parse(cls, url: str, *, map_instance: bool = True) -> ParsedDandiURL:
         """
-        Parse a Dandi Archive URL and return a `ParsedDandiURL` instance.
-
-        The accepted URL formats are as follows.  Text in [brackets] is
-        optional.  A ``server`` field is a base API, GUI, or redirector URL for
-        a registered Dandi Archive instance.  If an optional ``version`` field
-        is omitted from a URL, the given Dandiset's most recent published
-        version, if any, or else its draft version will be used.
-
-        - :samp:`https://identifiers.org/DANDI:{dandiset-id}[/{version}]`
-          (case insensitive; ``version`` cannot be "draft") when it redirects
-          to one of the other URL formats
-
-        - :samp:`DANDI:{dandiset-id}[/{version}]` — (case insensitive;
-          ``version`` cannot be "draft") Abbreviation for the above
-
-        - Any ``https://dandiarchive.org/`` or
-          ``https://*dandiarchive-org.netflify.app/`` URL which redirects to
-          one of the other URL formats
-
-        - :samp:`https://{server}[/api]/[#/]dandiset/{dandiset-id}[/{version}][/files[?location={path}]]`
-          — If ``path`` is not specified, the URL refers to a Dandiset and is
-          converted to a `DandisetURL`.  If ``path`` is specified and it ends
-          with a forward slash, the URL refers to an asset folder and is
-          converted to an `AssetFolderURL` instance; if it does not end with a
-          slash, it refers to a single asset and is converted to an
-          `AssetItemURL` instance.
-
-        - :samp:`https://{server}[/api]/dandisets/{dandiset-id}[/versions[/{version}]]`
-          — Refers to a Dandiset and is converted to a `DandisetURL`
-
-        - :samp:`https://{server}[/api]/assets/{asset-id}[/download]` — Refers
-          to a single asset by identifier and is converted to a
-          `BaseAssetIDURL`
-
-        - :samp:`https://{server}[/api]/dandisets/{dandiset-id}/versions/{version}/assets/{asset-id}[/download]`
-          — Refers to a single asset and is converted to an `AssetIDURL`
-
-        - :samp:`https://{server}[/api]/dandisets/{dandiset-id}/versions/{version}/assets/?path={path}`
-          — Refers to all assets in the given Dandiset whose paths begin with
-          the prefix ``path``; converted to an `AssetPathPrefixURL`
-
-        - :samp:`dandi://{instance-name}/{dandiset-id}[@{version}][/{path}]`,
-          where ``instance-name`` is the name of a registered Dandi Archive
-          instance.  If ``path`` is not specified, the URL refers to a Dandiset
-          and is converted to a `DandisetURL`.  If ``path`` is specified and it
-          ends with a forward slash, the URL refers to an asset folder and is
-          converted to an `AssetFolderURL` instance; if it does not end with a
-          slash, it refers to a single asset and is converted to an
-          `AssetItemURL` instance.
-
-        - Any other HTTPS URL that redirects to one of the above
+        Parse a Dandi Archive URL and return a `ParsedDandiURL` instance.  See
+        :ref:`resource_ids` for the supported URL formats.
 
         :raises UnknownURLError: if the URL is not one of the above
         """

docs/source/cmdline/dandi.rst

@@ -0,0 +1,25 @@
+:program:`dandi`
+================
+
+::
+
+    dandi [<global options>] <subcommand> [<arguments>]
+
+A command-line client for interacting with the `DANDI Archive
+<http://dandiarchive.org>`_.
+
+Global Options
+--------------
+
+.. option:: -l <level>, --log-level <level>
+
+    Set the `logging level`_ to the given value; default: ``INFO``.  The level
+    can be given as a case-insensitive level name or as a numeric value.
+
+    .. _logging level: https://docs.python.org/3/library/logging.html
+                       #logging-levels
+
+.. option:: --pdb
+
+    Handle errors by opening `pdb (the Python Debugger)
+    <https://docs.python.org/3/library/pdb.html>`_

docs/source/cmdline/delete.rst

@@ -5,12 +5,34 @@
 
     dandi [<global options>] delete [<options>] [<paths> ...]
 
-Delete dandisets and assets from the server.
+Delete Dandisets and assets from the server.
 
-PATH could be a local path or a URL to an asset, directory, or an entire
-dandiset.
+Each argument must be either a file path pointing to an asset file or directory
+in a local Dandiset (in which case the corresponding assets are deleted on the
+remote server) or a :ref:`resource identifier <resource_ids>` pointing to a
+remote asset, directory, or entire Dandiset.
 
 Options
 -------
 
+.. option:: -i, --instance <instance-name>
+
+    DANDI instance to delete assets & Dandisets from  [default: ``dandi``]
+
 .. option:: --skip-missing
+
+    By default, if an argument points to a remote resource that does not exist,
+    an error is raised.  If :option:`--skip-missing` is supplied, missing
+    resources are instead simply silently ignored.
+
+
+Development Options
+-------------------
+
+The following options are intended only for development & testing purposes.
+They are only available if the :envvar:`DANDI_DEVEL` environment variable is
+set to a nonempty value.
+
+.. option:: --devel-debug
+
+    Do not use pyout callbacks, do not swallow exceptions, do not parallelize.

docs/source/cmdline/digest.rst

@@ -12,4 +12,4 @@ Options
 
 .. option:: -d, --digest [dandi-etag|md5|sha1|sha256|sha512]
 
-    Digest algorithm to use  [default: dandi-etag]
+    Digest algorithm to use  [default: ``dandi-etag``]

docs/source/cmdline/download.rst

@@ -3,34 +3,43 @@
 
 ::
 
-    dandi [<global options>] download [<options>] [<url> ...]
+    dandi [<global options>] download [<options>] <url>
+
+Download a Dandiset, asset, or folder of assets from DANDI.
+
+See :ref:`resource_ids` for allowed URL formats.
 
 Options
 -------
 
-.. option:: -o, --output-dir <dir>
+.. option:: --download [dandiset.yaml,assets,all]
 
-    Directory where to download to (directory must exist).  Files will be
-    downloaded with paths relative to that directory.
+    Comma-separated list of elements to download  [default: ``all``]
 
 .. option:: -e, --existing [error|skip|overwrite|overwrite-different|refresh]
 
-    What to do if a file found existing locally. 'refresh': verify
-    that according to the size and mtime, it is the same file, if not -
-    download and overwrite.
+    How to handle paths that already exist locally  [default: ``error``]
+
+    For ``refresh``, if the local file's size and mtime are the same as on the
+    server, the asset is skipped; otherwise, it is redownloaded.
 
 .. option:: -f, --format [pyout|debug]
 
-    Choose the format/frontend for output.
+    Choose the format/frontend for output  [default: ``pyout``]
+
+.. option:: -i, --instance <instance-name>
+
+    DANDI instance to download from  [default: ``dandi``]
 
 .. option:: -J, --jobs <int>
 
-    Number of parallel download jobs.
+    Number of parallel download jobs  [default: 6]
 
-.. option:: --download [dandiset.yaml,assets,all]
+.. option:: -o, --output-dir <dir>
 
-    Comma-separated list of elements to download
+    Directory to download to (must exist).  Files will be downloaded with paths
+    relative to that directory.  [default: current working directory]
 
 .. option:: --sync
 
-    Delete local assets that do not exist on the server
+    Delete local assets that do not exist on the server after downloading

docs/source/cmdline/index.rst

@@ -5,4 +5,5 @@ Command-Line Interface
 .. toctree::
     :glob:
 
+    dandi
     *

docs/source/cmdline/ls.rst

@@ -5,46 +5,58 @@
 
     dandi [<global options>] ls [<options>] [<path|url> ...]
 
-List .nwb files and dandisets metadata.
-
-Patterns for known setups:
-
-- ``DANDI:<dandiset id>``
-- ``https://dandiarchive.org/...``
-- ``https://identifiers.org/DANDI:<dandiset id>``
-- ``https://<server>[/api]/[#/]dandiset/<dandiset id>[/<version>][/files[?location=<path>]]``
-- ``https://*dandiarchive-org.netflify.app/...``
-- ``https://<server>[/api]/dandisets/<dandiset id>[/versions[/<version>]]``
-- ``https://<server>[/api]/dandisets/<dandiset id>/versions/<version>/assets/<asset id>[/download]``
-- ``https://<server>[/api]/dandisets/<dandiset id>/versions/<version>/assets/?path=<path>``
-- ``dandi://<instance name>/<dandiset id>[@<version>][/<path>]``
-- ``https://<server>/...``
+List :file:`*.nwb` files' and Dandisets' metadata.
 
+The arguments may be either :ref:`resource identifiers <resource_ids>` or paths
+to local files/directories.
 
 Options
 -------
 
-.. option:: -F, --fields <fields>
-
-    Comma-separated list of fields to display.  An empty value to trigger a
-    list of available fields to be printed out
-
 .. option:: -f, --format [auto|pyout|json|json_pp|json_lines|yaml]
 
-    Choose the format/frontend for output. If 'auto' (default), 'pyout' will be
-    used in case of multiple files, and 'yaml' for a single file.
+    Choose the format/frontend for output.  If ``auto`` (the default),
+    ``pyout`` will be used in case of multiple files, ``yaml`` for a single
+    file.
 
-.. option:: -r, --recursive
+.. option:: -F, --fields <fields>
 
-    Recurse into content of dandisets/directories. Only .nwb files will be
-    considered.
+    Comma-separated list of fields to display.  Specifying ``-F ""`` causes a
+    list of available fields to be printed out.
 
 .. option:: -J, --jobs <int>
 
-    Number of parallel download jobs.
+    Number of parallel download jobs  [default: 6]
 
 .. option:: --metadata [api|all|assets]
 
+    Control when to include asset metadata for remote assets:
+
+    - ``api`` [default] — only include asset metadata if returned by the API
+      response (i.e., if a URL identifying an asset by ID was supplied)
+
+    - ``all`` — make an additional request to fetch asset metadata if not
+      returned by initial API response
+
+    - ``assets`` — same as ``all``
+
+.. option:: -r, --recursive
+
+    Recurse into Dandisets/directories.  Only :file:`*.nwb` files will be
+    considered.
+
 .. option:: --schema <version>
 
     Convert metadata to new schema version
+
+
+Development Options
+-------------------
+
+The following options are intended only for development & testing purposes.
+They are only available if the :envvar:`DANDI_DEVEL` environment variable is
+set to a nonempty value.
+
+.. option:: --use-fake-digest
+
+    Use dummy value for digests of local files instead of computing