Update for Pydantic v2

dandi/cli/cmd_download.py

@@ -9,7 +9,7 @@
 from ..dandiarchive import _dandi_url_parser, parse_dandi_url
 from ..dandiset import Dandiset
 from ..download import DownloadExisting, DownloadFormat, PathType
-from ..utils import get_instance
+from ..utils import get_instance, joinurl
 
 
 # The use of f-strings apparently makes this not a proper docstring, and so
@@ -131,9 +131,9 @@ def download(
                 pass
             else:
                 if instance.gui is not None:
-                    url = [f"{instance.gui}/#/dandiset/{dandiset_id}/draft"]
+                    url = [joinurl(instance.gui, f"/#/dandiset/{dandiset_id}/draft")]
                 else:
-                    url = [f"{instance.api}/dandisets/{dandiset_id}/"]
+                    url = [joinurl(instance.api, f"/dandisets/{dandiset_id}/")]
 
     return download.download(
         url,

dandi/cli/cmd_ls.py

@@ -96,8 +96,8 @@ def ls(
         all_fields = tuple(
             sorted(
                 set(common_fields)
-                | models.Dandiset.__fields__.keys()
-                | models.Asset.__fields__.keys()
+                | models.Dandiset.model_fields.keys()
+                | models.Asset.model_fields.keys()
             )
         )
     else:
@@ -345,7 +345,7 @@ def fn():
                             path,
                             schema_version=schema,
                             digest=Digest.dandi_etag(digest),
-                        ).json_dict()
+                        ).model_dump(mode="json", exclude_none=True)
                 else:
                     if path.endswith(tuple(ZARR_EXTENSIONS)):
                         if use_fake_digest:

dandi/cli/cmd_service_scripts.py

@@ -104,7 +104,7 @@ def reextract_metadata(url: str, diff: bool, when: str) -> None:
                 lgr.info("Extracting new metadata for asset")
                 metadata = nwb2asset(asset.as_readable(), digest=digest)
                 metadata.path = asset.path
-                mddict = metadata.json_dict()
+                mddict = metadata.model_dump(mode="json", exclude_none=True)
                 if diff:
                     oldmd = asset.get_raw_metadata()
                     oldmd_str = yaml_dump(oldmd)

dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json

@@ -386,7 +386,7 @@
             "includeInCitation": true
         }
     ],
-    "dateCreated": "2023-04-25T16:28:26.500181+00:00",
+    "dateCreated": "2023-04-25T16:28:26.500181Z",
     "description": "<jats:p>Progress in science requires standardized assays whose results can be readily shared, compared, and reproduced across laboratories. Reproducibility, however, has been a concern in neuroscience, particularly for measurements of mouse behavior. Here we show that a standardized task to probe decision-making in mice produces reproducible results across multiple laboratories. We designed a task for head-fixed mice that combines established assays of perceptual and value-based decision making, and we standardized training protocol and experimental hardware, software, and procedures. We trained 140 mice across seven laboratories in three countries, and we collected 5 million mouse choices into a publicly available database. Learning speed was variable across mice and laboratories, but once training was complete there were no significant differences in behavior across laboratories. Mice in different laboratories adopted similar reliance on visual stimuli, on past successes and failures, and on estimates of stimulus prior probability to guide their choices. These results reveal that a complex mouse behavior can be successfully reproduced across multiple laboratories. They establish a standard for reproducible rodent behavior, and provide an unprecedented dataset and open-access tools to study decision-making in mice. More generally, they indicate a path towards achieving reproducibility in neuroscience through collaborative open-science approaches.</jats:p>",
     "assetsSummary": {
         "schemaKey": "AssetsSummary",

dandi/cli/tests/data/update_dandiset_from_doi/elife.json

@@ -105,7 +105,7 @@
             "includeInCitation": true
         }
     ],
-    "dateCreated": "2023-04-25T16:28:30.453019+00:00",
+    "dateCreated": "2023-04-25T16:28:30.453019Z",
     "description": "<jats:p>Proprioception, the sense of body position, movement, and associated forces, remains poorly understood, despite its critical role in movement. Most studies of area 2, a proprioceptive area of somatosensory cortex, have simply compared neurons\u2019 activities to the movement of the hand through space. Using motion tracking, we sought to elaborate this relationship by characterizing how area 2 activity relates to whole arm movements. We found that a whole-arm model, unlike classic models, successfully predicted how features of neural activity changed as monkeys reached to targets in two workspaces. However, when we then evaluated this whole-arm model across active and passive movements, we found that many neurons did not consistently represent the whole arm over both conditions. These results suggest that 1) neural activity in area 2 includes representation of the whole arm during reaching and 2) many of these neurons represented limb state differently during active and passive movements.</jats:p>",
     "assetsSummary": {
         "schemaKey": "AssetsSummary",

dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json

@@ -45,7 +45,7 @@
             "includeInCitation": true
         }
     ],
-    "dateCreated": "2023-04-25T16:28:28.308094+00:00",
+    "dateCreated": "2023-04-25T16:28:28.308094Z",
     "description": "<jats:p>Reinforcement learning theory plays a key role in understanding the behavioral and neural mechanisms of choice behavior in animals and humans. Especially, intermediate variables of learning models estimated from behavioral data, such as the expectation of reward for each candidate choice (action value), have been used in searches for the neural correlates of computational elements in learning and decision making. The aims of the present study are as follows: (1) to test which computational model best captures the choice learning process in animals and (2) to elucidate how action values are represented in different parts of the corticobasal ganglia circuit. We compared different behavioral learning algorithms to predict the choice sequences generated by rats during a free-choice task and analyzed associated neural activity in the nucleus accumbens (NAc) and ventral pallidum (VP). The major findings of this study were as follows: (1) modified versions of an action\u2013value learning model captured a variety of choice strategies of rats, including win-stay\u2013lose-switch and persevering behavior, and predicted rats' choice sequences better than the best multistep Markov model; and (2) information about action values and future actions was coded in both the NAc and VP, but was less dominant than information about trial types, selected actions, and reward outcome. The results of our model-based analysis suggest that the primary role of the NAc and VP is to monitor information important for updating choice behaviors. Information represented in the NAc and VP might contribute to a choice mechanism that is situated elsewhere.</jats:p>",
     "assetsSummary": {
         "schemaKey": "AssetsSummary",

dandi/cli/tests/data/update_dandiset_from_doi/nature.json

@@ -46,7 +46,7 @@
             "includeInCitation": true
         }
     ],
-    "dateCreated": "2023-04-25T16:28:31.601155+00:00",
+    "dateCreated": "2023-04-25T16:28:31.601155Z",
     "description": "<jats:title>Abstract</jats:title><jats:p>Spatial cognition depends on an accurate representation of orientation within an environment. Head direction cells in distributed brain regions receive a range of sensory inputs, but visual input is particularly important for aligning their responses to environmental landmarks. To investigate how population-level heading responses are aligned to visual input, we recorded from retrosplenial cortex (RSC) of head-fixed mice in a moving environment using two-photon calcium imaging. We show that RSC neurons are tuned to the animal\u2019s relative orientation in the environment, even in the absence of head movement. Next, we found that RSC receives functionally distinct projections from visual and thalamic areas and contains several functional classes of neurons. While some functional classes mirror RSC inputs, a newly discovered class coregisters visual and thalamic signals. Finally, decoding analyses reveal unique contributions to heading from each class. Our results suggest an RSC circuit for anchoring heading representations to environmental visual landmarks.</jats:p>",
     "assetsSummary": {
         "schemaKey": "AssetsSummary",

dandi/cli/tests/data/update_dandiset_from_doi/neuron.json

@@ -45,7 +45,7 @@
             "includeInCitation": true
         }
     ],
-    "dateCreated": "2023-04-25T16:28:29.373034+00:00",
+    "dateCreated": "2023-04-25T16:28:29.373034Z",
     "description": "A test Dandiset",
     "assetsSummary": {
         "schemaKey": "AssetsSummary",

dandi/dandiapi.py

@@ -13,7 +13,7 @@
 import re
 from time import sleep, time
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, FrozenSet, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 from urllib.parse import quote_plus, urlparse, urlunparse
 
 import click
@@ -44,6 +44,7 @@
     get_instance,
     is_interactive,
     is_page2_url,
+    joinurl,
 )
 
 if TYPE_CHECKING:
@@ -285,16 +286,12 @@ def request(
     def get_url(self, path: str) -> str:
         """
         Append a slash-separated ``path`` to the instance's base URL.  The two
-        components are separated by a single slash, and any trailing slashes
-        are removed.
+        components are separated by a single slash, removing any excess slashes
+        that would be present after naïve concatenation.
 
         If ``path`` is already an absolute URL, it is returned unchanged.
         """
-        # Construct the url
-        if path.lower().startswith(("http://", "https://")):
-            return path
-        else:
-            return self.api_url.rstrip("/") + "/" + path.lstrip("/")
+        return joinurl(self.api_url, path)
 
     def get(self, path: str, **kwargs: Any) -> Any:
         """
@@ -614,29 +611,21 @@ def get_asset(self, asset_id: str) -> BaseRemoteAsset:
         return BaseRemoteAsset.from_base_data(self, info, metadata)
 
 
-class APIBase(BaseModel):
+# `arbitrary_types_allowed` is needed for `client: DandiAPIClient`
+class APIBase(BaseModel, populate_by_name=True, arbitrary_types_allowed=True):
     """
     Base class for API objects implemented in pydantic.
 
     This class (aside from the `json_dict()` method) is an implementation
     detail; do not rely on it.
     """
 
-    JSON_EXCLUDE: ClassVar[FrozenSet[str]] = frozenset(["client"])
-
     def json_dict(self) -> dict[str, Any]:
         """
         Convert to a JSONable `dict`, omitting the ``client`` attribute and
         using the same field names as in the API
         """
-        data = json.loads(self.json(exclude=self.JSON_EXCLUDE, by_alias=True))
-        assert isinstance(data, dict)
-        return data
-
-    class Config:
-        allow_population_by_field_name = True
-        # To allow `client: Session`:
-        arbitrary_types_allowed = True
+        return self.model_dump(mode="json", by_alias=True)
 
 
 class Version(APIBase):
@@ -710,7 +699,7 @@ class RemoteDandisetData(APIBase):
     modified: datetime
     contact_person: str
     embargo_status: EmbargoStatus
-    most_recent_published_version: Optional[Version]
+    most_recent_published_version: Optional[Version] = None
     draft_version: Version
 
 
@@ -752,7 +741,7 @@ def __init__(
             self._version = version
         self._data: RemoteDandisetData | None
         if data is not None:
-            self._data = RemoteDandisetData.parse_obj(data)
+            self._data = RemoteDandisetData.model_validate(data)
         else:
             self._data = None
 
@@ -762,7 +751,7 @@ def __str__(self) -> str:
     def _get_data(self) -> RemoteDandisetData:
         if self._data is None:
             try:
-                self._data = RemoteDandisetData.parse_obj(
+                self._data = RemoteDandisetData.model_validate(
                     self.client.get(f"/dandisets/{self.identifier}/")
                 )
             except HTTP404Error:
@@ -875,9 +864,9 @@ def from_data(cls, client: DandiAPIClient, data: dict[str, Any]) -> RemoteDandis
         when acquiring data using means outside of this library.
         """
         if data.get("most_recent_published_version") is not None:
-            version = Version.parse_obj(data["most_recent_published_version"])
+            version = Version.model_validate(data["most_recent_published_version"])
         else:
-            version = Version.parse_obj(data["draft_version"])
+            version = Version.model_validate(data["draft_version"])
         return cls(
             client=client, identifier=data["identifier"], version=version, data=data
         )
@@ -917,7 +906,7 @@ def get_versions(self, order: str | None = None) -> Iterator[Version]:
             for v in self.client.paginate(
                 f"{self.api_path}versions/", params={"order": order}
             ):
-                yield Version.parse_obj(v)
+                yield Version.model_validate(v)
         except HTTP404Error:
             raise NotFoundError(f"No such Dandiset: {self.identifier!r}")
 
@@ -932,7 +921,7 @@ def get_version(self, version_id: str) -> VersionInfo:
             `Version`.
         """
         try:
-            return VersionInfo.parse_obj(
+            return VersionInfo.model_validate(
                 self.client.get(
                     f"/dandisets/{self.identifier}/versions/{version_id}/info/"
                 )
@@ -978,7 +967,7 @@ def get_metadata(self) -> models.Dandiset:
             metadata.  Consider using `get_raw_metadata()` instead in order to
             fetch unstructured, possibly-invalid metadata.
         """
-        return models.Dandiset.parse_obj(self.get_raw_metadata())
+        return models.Dandiset.model_validate(self.get_raw_metadata())
 
     def get_raw_metadata(self) -> dict[str, Any]:
         """
@@ -996,7 +985,7 @@ def set_metadata(self, metadata: models.Dandiset) -> None:
         """
         Set the metadata for this version of the Dandiset to the given value
         """
-        self.set_raw_metadata(metadata.json_dict())
+        self.set_raw_metadata(metadata.model_dump(mode="json", exclude_none=True))
 
     def set_raw_metadata(self, metadata: dict[str, Any]) -> None:
         """
@@ -1049,7 +1038,7 @@ def publish(self, max_time: float = 120) -> RemoteDandiset:
         )
         start = time()
         while time() - start < max_time:
-            v = Version.parse_obj(self.client.get(f"{draft_api_path}info/"))
+            v = Version.model_validate(self.client.get(f"{draft_api_path}info/"))
             if v.status is VersionStatus.PUBLISHED:
                 break
             sleep(0.5)
@@ -1273,7 +1262,7 @@ class BaseRemoteAsset(ABC, APIBase):
 
     #: The `DandiAPIClient` instance that returned this `BaseRemoteAsset`
     #: and which the latter will use for API requests
-    client: DandiAPIClient
+    client: DandiAPIClient = Field(exclude=True)
     #: The asset identifier
     identifier: str = Field(alias="asset_id")
     #: The asset's (forward-slash-separated) path
@@ -1294,6 +1283,15 @@ def __init__(self, **data: Any) -> None:  # type: ignore[no-redef]
         # underscores, so we have to do it ourselves.
         self._metadata = data.get("metadata", data.get("_metadata"))
 
+    def __eq__(self, other: Any) -> bool:
+        if type(self) is type(other):
+            # dict() includes fields with `exclude=True` (which are absent from
+            # the return value of `model_dump()`) but not private fields.  We
+            # want to compare the former but not the latter.
+            return dict(self) == dict(other)
+        else:
+            return NotImplemented
+
     def __str__(self) -> str:
         return f"{self.client._instance_id}:assets/{self.identifier}"
 
@@ -1360,7 +1358,7 @@ def get_metadata(self) -> models.Asset:
             valid metadata.  Consider using `get_raw_metadata()` instead in
             order to fetch unstructured, possibly-invalid metadata.
         """
-        return models.Asset.parse_obj(self.get_raw_metadata())
+        return models.Asset.model_validate(self.get_raw_metadata())
 
     def get_raw_metadata(self) -> dict[str, Any]:
         """Fetch the metadata for the asset as an unprocessed `dict`"""
@@ -1610,7 +1608,7 @@ def iterfiles(self, prefix: str | None = None) -> Iterator[RemoteZarrEntry]:
         for r in self.client.paginate(
             f"{self.client.api_url}/zarr/{self.zarr}/files", params={"prefix": prefix}
         ):
-            data = ZarrEntryServerData.parse_obj(r)
+            data = ZarrEntryServerData.model_validate(r)
             yield RemoteZarrEntry.from_server_data(self, data)
 
     def get_entry_by_path(self, path: str) -> RemoteZarrEntry:
@@ -1667,13 +1665,12 @@ class RemoteAsset(BaseRemoteAsset):
     `RemoteDandiset`.
     """
 
-    JSON_EXCLUDE = frozenset(["client", "dandiset_id", "version_id"])
-
     #: The identifier for the Dandiset to which the asset belongs
-    dandiset_id: str
+    dandiset_id: str = Field(exclude=True)
+
     #: The identifier for the version of the Dandiset to which the asset
     #: belongs
-    version_id: str
+    version_id: str = Field(exclude=True)
 
     @classmethod
     def from_data(
@@ -1738,7 +1735,9 @@ def set_metadata(self, metadata: models.Asset) -> None:
         Set the metadata for the asset to the given value and update the
         `RemoteAsset` in place.
         """
-        return self.set_raw_metadata(metadata.json_dict())
+        return self.set_raw_metadata(
+            metadata.model_dump(mode="json", exclude_none=True)
+        )
 
     @abstractmethod
     def set_raw_metadata(self, metadata: dict[str, Any]) -> None:

dandi/dandiarchive.py

@@ -37,7 +37,7 @@
 from typing import Any
 from urllib.parse import unquote as urlunquote
 
-from pydantic import AnyHttpUrl, parse_obj_as
+from pydantic import AnyHttpUrl, TypeAdapter
 import requests
 
 from . import get_logger
@@ -82,9 +82,8 @@ class ParsedDandiURL(ABC):
     def api_url(self) -> AnyHttpUrl:
         """The base URL of the Dandi API service, without a trailing slash"""
         # Kept for backwards compatibility
-        r = parse_obj_as(AnyHttpUrl, self.instance.api.rstrip("/"))
-        assert isinstance(r, AnyHttpUrl)
-        return r  # type: ignore[no-any-return]
+        adapter = TypeAdapter(AnyHttpUrl)
+        return adapter.validate_python(self.instance.api.rstrip("/"))
 
     def get_client(self) -> DandiAPIClient:
         """

dandi/files/bases.py

@@ -98,7 +98,7 @@ def get_metadata(
         """Return the Dandiset metadata inside the file"""
         with open(self.filepath) as f:
             meta = yaml_load(f, typ="safe")
-        return DandisetMeta.unvalidated(**meta)
+        return DandisetMeta.model_construct(**meta)
 
     # TODO: @validate_cache.memoize_path
     def get_validation_errors(
@@ -184,7 +184,7 @@ def get_validation_errors(
             )
         try:
             asset = self.get_metadata(digest=self._DUMMY_DIGEST)
-            BareAsset(**asset.dict())
+            BareAsset(**asset.model_dump())
         except ValidationError as e:
             if devel_debug:
                 raise

dandi/files/bids.py

@@ -94,7 +94,9 @@ def _validate(self) -> None:
                 )
                 # Don't apply eta-reduction to the lambda, as mypy needs to be
                 # assured that defaultdict's argument takes no parameters.
-                self._asset_metadata = defaultdict(lambda: BareAsset.unvalidated())
+                self._asset_metadata = defaultdict(
+                    lambda: BareAsset.model_construct()  # type: ignore[call-arg]
+                )
                 for result in results:
                     if result.id in BIDS_ASSET_ERRORS:
                         assert result.path
@@ -230,7 +232,10 @@ def get_metadata(
         bids_metadata = BIDSAsset.get_metadata(self, digest, ignore_errors)
         nwb_metadata = NWBAsset.get_metadata(self, digest, ignore_errors)
         return BareAsset(
-            **{**bids_metadata.dict(), **nwb_metadata.dict(exclude_none=True)}
+            **{
+                **bids_metadata.model_dump(),
+                **nwb_metadata.model_dump(exclude_none=True),
+            }
         )