From 215f59bac2ef2ffb469403c20d62edfecd3b0f2b Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 18 Jun 2024 11:06:51 +0200 Subject: [PATCH 1/2] correctly use base mapping in FullMeshRequest when forwarding from TS to DS --- .../controllers/DSMeshController.scala | 1 + .../services/MeshMappingHelper.scala | 86 ++++++++++--------- .../TSRemoteDatastoreClient.scala | 15 ++-- .../tracings/volume/TSFullMeshService.scala | 3 +- 4 files changed, 55 insertions(+), 50 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala index b061b016137..0052faaab93 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala @@ -44,6 +44,7 @@ class DSMeshController @Inject()( the oversegmentation. Collect mesh chunks of all *unmapped* segment ids belonging to the supplied agglomerate id. If it is not set, use meshfile as is, assume passed id is present in meshfile + Note: in case of an editable mapping, targetMappingName is its baseMapping name. */ targetMappingName: Option[String], editableMappingTracingId: Option[String]): Action[ListMeshChunksRequest] = diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala index 3389fbbd253..78e755c4d0d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala @@ -23,55 +23,57 @@ trait MeshMappingHelper { mappingNameForMeshFile: Option[String], omitMissing: Boolean, // If true, failing lookups in the agglomerate file will just return empty list. token: Option[String])(implicit ec: ExecutionContext): Fox[List[Long]] = - targetMappingName match { - case None => + (targetMappingName, editableMappingTracingId) match { + case (None, None) => // No mapping selected, assume id matches meshfile Fox.successful(List(agglomerateId)) - case Some(mappingName) if mappingNameForMeshFile.contains(mappingName) => + case (Some(mappingName), None) if mappingNameForMeshFile.contains(mappingName) => // Mapping selected, but meshfile has the same mapping name in its metadata, assume id matches meshfile Fox.successful(List(agglomerateId)) - case Some(mappingName) => + case (Some(mappingName), None) => // Mapping selected, but meshfile does not have matching mapping name in its metadata, // assume agglomerate id, fetch oversegmentation segment ids for it - val agglomerateFileKey = AgglomerateFileKey( - organizationName, - datasetName, - dataLayerName, - mappingName - ) - editableMappingTracingId match { - case Some(tracingId) => - for { - tracingstoreUri <- dsRemoteWebknossosClient.getTracingstoreUri - segmentIdsResult <- dsRemoteTracingstoreClient.getEditableMappingSegmentIdsForAgglomerate(tracingstoreUri, - tracingId, - agglomerateId, - token) - segmentIds <- if (segmentIdsResult.agglomerateIdIsPresent) - Fox.successful(segmentIdsResult.segmentIds) - else - for { - agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox - localSegmentIds <- agglomerateService.segmentIdsForAgglomerateId( - agglomerateFileKey, - agglomerateId - ) - } yield localSegmentIds - } yield segmentIds - case _ => + for { + agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox + segmentIdsBox <- agglomerateService + .segmentIdsForAgglomerateId( + AgglomerateFileKey( + organizationName, + datasetName, + dataLayerName, + mappingName + ), + agglomerateId + ) + .futureBox + segmentIds <- segmentIdsBox match { + case Full(segmentIds) => Fox.successful(segmentIds) + case _ => if (omitMissing) Fox.successful(List.empty) else segmentIdsBox.toFox + } + } yield segmentIds + case (Some(mappingName), Some(tracingId)) => + for { + tracingstoreUri <- dsRemoteWebknossosClient.getTracingstoreUri + segmentIdsResult <- dsRemoteTracingstoreClient.getEditableMappingSegmentIdsForAgglomerate(tracingstoreUri, + tracingId, + agglomerateId, + token) + segmentIds <- if (segmentIdsResult.agglomerateIdIsPresent) + Fox.successful(segmentIdsResult.segmentIds) + else // the agglomerate id is not present in the editable mapping. Fetch its info from the base mapping. for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox - segmentIdsBox <- agglomerateService - .segmentIdsForAgglomerateId( - agglomerateFileKey, - agglomerateId - ) - .futureBox - segmentIds <- segmentIdsBox match { - case Full(segmentIds) => Fox.successful(segmentIds) - case _ => if (omitMissing) Fox.successful(List.empty) else segmentIdsBox.toFox - } - } yield segmentIds - } + localSegmentIds <- agglomerateService.segmentIdsForAgglomerateId( + AgglomerateFileKey( + organizationName, + datasetName, + dataLayerName, + mappingName + ), + agglomerateId + ) + } yield localSegmentIds + } yield segmentIds + case _ => Fox.failure("Cannot determine segment ids for editable mapping without base mapping") } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala index 256b0886d40..c2728ac4a1c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala @@ -133,7 +133,7 @@ class TSRemoteDatastoreClient @Inject()( def querySegmentIndex(remoteFallbackLayer: RemoteFallbackLayer, segmentId: Long, mag: Vec3Int, - mappingName: Option[String], + mappingName: Option[String], // should be the baseMappingName in case of editable mappings editableMappingTracingId: Option[String], userToken: Option[String]): Fox[Seq[Vec3Int]] = for { @@ -152,12 +152,13 @@ class TSRemoteDatastoreClient @Inject()( indices = positions.map(_.scale(1f / DataLayer.bucketLength)) // Route returns positions to use the same interface as tracing store, we want indices } yield indices - def querySegmentIndexForMultipleSegments(remoteFallbackLayer: RemoteFallbackLayer, - segmentIds: Seq[Long], - mag: Vec3Int, - mappingName: Option[String], - editableMappingTracingId: Option[String], - userToken: Option[String]): Fox[Seq[(Long, Seq[Vec3Int])]] = + def querySegmentIndexForMultipleSegments( + remoteFallbackLayer: RemoteFallbackLayer, + segmentIds: Seq[Long], + mag: Vec3Int, + mappingName: Option[String], // should be the baseMappingName in case of editable mappings + editableMappingTracingId: Option[String], + userToken: Option[String]): Fox[Seq[(Long, Seq[Vec3Int])]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) result <- rpc(s"$remoteLayerUri/segmentIndex") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index 7a9c3dd2a1b..47ea985db17 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -44,8 +44,9 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[Array[Byte]] = for { remoteFallbackLayer <- remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + baseMappingName <- volumeTracingService.baseMappingName(tracing) fullMeshRequestAdapted = if (tracing.mappingIsEditable.getOrElse(false)) - fullMeshRequest.copy(mappingName = tracing.mappingName, + fullMeshRequest.copy(mappingName = baseMappingName, editableMappingTracingId = Some(tracingId), mappingType = Some("HDF5")) else fullMeshRequest From 247a4633228d63d0821e1cf42da07ef56c2c443f Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 18 Jun 2024 11:14:51 +0200 Subject: [PATCH 2/2] changelog --- CHANGELOG.unreleased.md | 1 + .../webknossos/datastore/services/MeshMappingHelper.scala | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 6f9a0636925..cc0e3dedbf2 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -32,6 +32,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Fixed a bug where sometimes old mismatching javascript code would be served after upgrades. [#7854](https://github.com/scalableminds/webknossos/pull/7854) - Fixed a bug where dataset uploads of zipped tiff data via the UI would be rejected. [#7856](https://github.com/scalableminds/webknossos/pull/7856) - Fixed a bug with incorrect valiation of layer names in the animation modal. [#7882](https://github.com/scalableminds/webknossos/pull/7882) +- Fixed a bug in the fullMesh.stl route used by the render_animation worker job, where some meshes in proofreading annotations could not be loaded. [#7887](https://github.com/scalableminds/webknossos/pull/7887) ### Removed - If the datasource-properties.json file for a dataset is missing or contains errors, WEBKNOSSOS no longer attempts to guess its contents from the raw data. Exploring remote datasets will still create the file. [#7697](https://github.com/scalableminds/webknossos/pull/7697) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala index 78e755c4d0d..1f03f1bb09b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala @@ -52,6 +52,8 @@ trait MeshMappingHelper { } } yield segmentIds case (Some(mappingName), Some(tracingId)) => + // An editable mapping tracing id is supplied. Ask the tracingstore for the segment ids. If it doesn’t know, + // use the mappingName (here the editable mapping’s base mapping) to look it up from file. for { tracingstoreUri <- dsRemoteWebknossosClient.getTracingstoreUri segmentIdsResult <- dsRemoteTracingstoreClient.getEditableMappingSegmentIdsForAgglomerate(tracingstoreUri,