Display dataset used storage (#7677)

* include used storage in dataset json * remove unused imports * snapshots * rudimentary display of size in dashboard * hide if < 10k * remove unused * Update frontend/javascripts/types/api_flow_types.ts Co-authored-by: Norman Rzepka <code@normanrz.com> * instead of size in bytes, show hint about linked layers in tooltip --------- Co-authored-by: Norman Rzepka <code@normanrz.com>
scalableminds · Mar 7, 2024 · 5222a04 · 5222a04
1 parent b59086e
commit 5222a04
Show file tree

Hide file tree

Showing 8 changed files with 27 additions and 4 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added support for uploading N5 and Neuroglancer Precomputed datasets. [#7578](https://github.com/scalableminds/webknossos/pull/7578)
 - Webknossos can now open ND Zarr datasets with arbitrary axis orders (not limited to `**xyz` anymore). [#7592](https://github.com/scalableminds/webknossos/pull/7592)
 - Added a new "Split from all neighboring segments" feature for the proofreading mode. [#7611](https://github.com/scalableminds/webknossos/pull/7611)
+- If storage scan is enabled, the measured used storage is now displayed in the dashboard’s dataset detail view. [#7677](https://github.com/scalableminds/webknossos/pull/7677)
 
 ### Changed
 - Datasets stored in WKW format are no longer loaded with memory mapping, reducing memory demands. [#7528](https://github.com/scalableminds/webknossos/pull/7528)

diff --git a/app/models/dataset/DatasetService.scala b/app/models/dataset/DatasetService.scala
@@ -15,7 +15,6 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.typesafe.scalalogging.LazyLogging
 import models.folder.FolderDAO
-import models.job.JobService
 import models.organization.{Organization, OrganizationDAO}
 import models.team._
 import models.user.{User, UserService}
@@ -33,7 +32,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                                datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO,
                                datasetDataLayerDAO: DatasetLayerDAO,
                                teamDAO: TeamDAO,
-                               jobService: JobService,
                                folderDAO: FolderDAO,
                                dataStoreService: DataStoreService,
                                teamService: TeamService,
@@ -333,6 +331,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       lastUsedByUser <- lastUsedTimeFor(dataset._id, requestingUserOpt) ?~> "dataset.list.fetchLastUsedTimeFailed"
       dataStoreJs <- dataStoreService.publicWrites(dataStore) ?~> "dataset.list.dataStoreWritesFailed"
       dataSource <- dataSourceFor(dataset, Some(organization)) ?~> "dataset.list.fetchDataSourceFailed"
+      usedStorageBytes <- Fox.runIf(requestingUserOpt.exists(u => u._organization == dataset._organization))(
+        organizationDAO.getUsedStorageForDataset(dataset._id))
     } yield {
       Json.obj(
         "name" -> dataset.name,
@@ -355,7 +355,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         "tags" -> dataset.tags,
         "folderId" -> dataset._folder,
         // included temporarily for compatibility with webknossos-libs, until a better versioning mechanism is implemented
-        "publication" -> None
+        "publication" -> None,
+        "usedStorageBytes" -> usedStorageBytes
       )
     }
 }
diff --git a/app/models/organization/Organization.scala b/app/models/organization/Organization.scala
@@ -188,6 +188,13 @@ class OrganizationDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionCont
       firstRow <- rows.headOption
     } yield firstRow
 
+  def getUsedStorageForDataset(datasetId: ObjectId): Fox[Long] =
+    for {
+      rows <- run(
+        q"SELECT SUM(usedStorageBytes) FROM webknossos.organization_usedStorage WHERE _dataset = $datasetId".as[Long])
+      firstRow <- rows.headOption
+    } yield firstRow
+
   def findNotRecentlyScanned(rescanInterval: FiniteDuration, limit: Int): Fox[List[Organization]] =
     for {
       rows <- run(q"""

diff --git a/frontend/javascripts/dashboard/folders/details_sidebar.tsx b/frontend/javascripts/dashboard/folders/details_sidebar.tsx
@@ -6,7 +6,7 @@ import {
   LoadingOutlined,
 } from "@ant-design/icons";
 import { Result, Spin, Tag, Tooltip } from "antd";
-import { stringToColor } from "libs/format_utils";
+import { stringToColor, formatCountToDataAmountUnit } from "libs/format_utils";
 import { pluralize } from "libs/utils";
 import _ from "lodash";
 import {
@@ -173,12 +173,22 @@ function DatasetDetails({ selectedDataset }: { selectedDataset: APIDatasetCompac
           )}
         </div>
       </Spin>
+
       {selectedDataset.isActive ? (
         <div style={{ marginBottom: 4 }}>
           <div className="sidebar-label">Tags</div>
           <DatasetTags dataset={selectedDataset} updateDataset={context.updateCachedDataset} />
         </div>
       ) : null}
+
+      {fullDataset?.usedStorageBytes && fullDataset.usedStorageBytes > 10000 ? (
+        <div style={{ marginBottom: 4 }}>
+          <div className="sidebar-label">Used Storage</div>
+          <Tooltip title="Note that linked and remote layers aren’t measured." placement="left">
+            <div>{formatCountToDataAmountUnit(fullDataset.usedStorageBytes, true)}</div>
+          </Tooltip>
+        </div>
+      ) : null}
     </>
   );
 }

diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts
@@ -147,6 +147,7 @@ const defaultState: OxalisState = {
     lastUsedByUser: 0,
     sortingKey: 123,
     publication: null,
+    usedStorageBytes: null,
   },
   tracing: {
     ...initialAnnotationInfo,

diff --git a/...napshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md b/...napshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md
@@ -294,6 +294,7 @@ Generated by [AVA](https://avajs.dev).
         publication: null,
         sortingKey: 1508495293763,
         tags: [],
+        usedStorageBytes: 0,
       },
       {
         allowedTeams: [
@@ -432,6 +433,7 @@ Generated by [AVA](https://avajs.dev).
         publication: null,
         sortingKey: 1508495293789,
         tags: [],
+        usedStorageBytes: 0,
       },
     ]
 

diff --git a/...s/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap b/...s/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap
diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts
@@ -181,6 +181,7 @@ type MutableAPIDatasetBase = MutableAPIDatasetId & {
   owningOrganization: string;
   publication: null | undefined;
   tags: Array<string>;
+  usedStorageBytes: number | null;
 };
 type APIDatasetBase = Readonly<MutableAPIDatasetBase>;
 export type MutableAPIDataset = MutableAPIDatasetBase & {