Merge pull request #388 from dandi/upload-sha256

Support passing precomputed file digest to DandiAPIClient upload methods
dandi · Feb 16, 2021 · 48679c6 · 48679c6
2 parents ea86eee + bab2056
commit 48679c6
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 6 deletions.
diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py
@@ -348,7 +348,9 @@ def _migrate_dandiset_metadata(cls, dandiset):
             dandiset["metadata"] = dandiset_metadata.pop("dandiset")
         return dandiset
 
-    def upload(self, dandiset_id, version_id, asset_metadata, filepath):
+    def upload(
+        self, dandiset_id, version_id, asset_metadata, filepath, sha256_digest=None
+    ):
         """
         Parameters
         ----------
@@ -362,12 +364,22 @@ def upload(self, dandiset_id, version_id, asset_metadata, filepath):
           the server.
         filepath: str or PathLike
           the path to the local file to upload
+        sha256_digest: str, optional
+          a precalculated SHA 256 digest for the file
         """
-        for r in self.iter_upload(dandiset_id, version_id, asset_metadata, filepath):
+        for r in self.iter_upload(
+            dandiset_id,
+            version_id,
+            asset_metadata,
+            filepath,
+            sha256_digest=sha256_digest,
+        ):
             if r["status"] == "validating":
                 sleep(0.1)
 
-    def iter_upload(self, dandiset_id, version_id, asset_metadata, filepath):
+    def iter_upload(
+        self, dandiset_id, version_id, asset_metadata, filepath, sha256_digest=None
+    ):
         """
         Parameters
         ----------
@@ -381,13 +393,21 @@ def iter_upload(self, dandiset_id, version_id, asset_metadata, filepath):
           the server.
         filepath: str or PathLike
           the path to the local file to upload
+        sha256_digest: str, optional
+          a precalculated SHA 256 digest for the file
 
         Returns
         -------
         a generator of `dict`s containing at least a ``"status"`` key
         """
-        filehash = Digester(["sha256"])(filepath)["sha256"]
-        lgr.debug("Calculated sha256 digest of %s for %s", filehash, filepath)
+        if sha256_digest is not None:
+            filehash = sha256_digest
+            lgr.debug(
+                "Using precalculated sha256 digest of %s for %s", filehash, filepath
+            )
+        else:
+            filehash = Digester(["sha256"])(filepath)["sha256"]
+            lgr.debug("Calculated sha256 digest of %s for %s", filehash, filepath)
         try:
             self.post("/uploads/validate/", json={"sha256": filehash})
         except requests.HTTPError as e:

diff --git a/dandi/upload.py b/dandi/upload.py
@@ -810,7 +810,9 @@ def process_path(path, relpath):
                 lgr.info("Replacing asset %s", extant["uuid"])
                 client.delete_asset(ds_identifier, "draft", extant["uuid"])
             yield {"status": "uploading"}
-            for r in client.iter_upload(ds_identifier, "draft", metadata, str(path)):
+            for r in client.iter_upload(
+                ds_identifier, "draft", metadata, str(path), sha256_digest=sha256_digest
+            ):
                 if r["status"] == "uploading":
                     uploaded_paths[str(path)]["size"] = r["current"]
                 yield r