feat: Updates to the alignment entity (#298)

* Updates to schema and template * Fixing tests * Adding files to alignments metadata * Update the alignment configs method type * Fixing typo * Adding undefined alignment_method_type
chanzuckerberg · Oct 3, 2024 · c5770ea · c5770ea
1 parent dd5f547
commit c5770ea
Show file tree

Hide file tree

Showing 25 changed files with 310 additions and 146 deletions.
diff --git a/ingestion_tools/dataset_configs/10000.yaml b/ingestion_tools/dataset_configs/10000.yaml
@@ -1,6 +1,7 @@
 alignments:
 - metadata:
     format: IMOD
+    method_type: patch_tracking
   sources:
   - source_multi_glob:
       list_globs:
@@ -62,7 +63,6 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_fas.csv
-
       is_visualization_default: false
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -85,7 +85,6 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_cyto_ribosomes.csv
-
       is_visualization_default: true
 - metadata:
     annotation_method: spectrum equalization filter + 2D CNN prediction + manual correction
@@ -106,7 +105,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 1
 - metadata:
@@ -128,7 +126,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: true
       mask_label: 2
 - metadata:
@@ -150,7 +147,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 3
 - metadata:
@@ -172,7 +168,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: true
       mask_label: 4
 - metadata:
@@ -194,7 +189,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: true
       mask_label: 5
 - metadata:
@@ -216,7 +210,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: true
       mask_label: 6
 - metadata:
@@ -238,7 +231,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 7
 - metadata:
@@ -260,7 +252,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 8
 - metadata:
@@ -282,7 +273,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 9
 - metadata:
@@ -304,7 +294,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: true
       mask_label: 10
 - metadata:
@@ -326,7 +315,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 11
 - metadata:
@@ -348,7 +336,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/180426_{mapped_segmask_name}.labels.mrc
-
       is_visualization_default: false
       mask_label: 13
 - metadata:
@@ -370,7 +357,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_membranes.mrc
-
       is_visualization_default: false
       mask_label: 1
 collection_metadata:

diff --git a/ingestion_tools/dataset_configs/10001.yaml b/ingestion_tools/dataset_configs/10001.yaml
@@ -1,6 +1,7 @@
 alignments:
 - metadata:
     format: IMOD
+    method_type: patch_tracking
   sources:
   - source_multi_glob:
       list_globs:
@@ -62,7 +63,6 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_fas.csv
-
       is_visualization_default: false
 - metadata:
     annotation_method: Cumulative template-matching trained 2D CNN predictions + visual
@@ -85,7 +85,6 @@ annotations:
       columns: xyz
       file_format: csv
       glob_string: particle_lists/{run_name}_cyto_ribosomes.csv
-
       is_visualization_default: true
 - metadata:
     annotation_method: manual segmentation
@@ -105,7 +104,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 1
 - metadata:
@@ -126,7 +124,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: true
       mask_label: 2
 - metadata:
@@ -147,7 +144,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 3
 - metadata:
@@ -168,7 +164,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: true
       mask_label: 4
 - metadata:
@@ -189,7 +184,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: true
       mask_label: 5
 - metadata:
@@ -210,7 +204,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: true
       mask_label: 6
 - metadata:
@@ -231,7 +224,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 7
 - metadata:
@@ -252,7 +244,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 8
 - metadata:
@@ -273,7 +264,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 9
 - metadata:
@@ -294,7 +284,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: true
       mask_label: 10
 - metadata:
@@ -315,7 +304,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 11
 - metadata:
@@ -336,7 +324,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_organelles.mrc
-
       is_visualization_default: false
       mask_label: 13
 - metadata:
@@ -358,7 +345,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_membranes.mrc
-
       is_visualization_default: false
       mask_label: 1
 collection_metadata:

diff --git a/ingestion_tools/dataset_configs/10002.yaml b/ingestion_tools/dataset_configs/10002.yaml
@@ -18,6 +18,7 @@ alignments:
       - 625
       - 1
     format: IMOD
+    method_type: patch_tracking
   sources:
   - source_multi_glob:
       list_globs:
@@ -75,7 +76,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: labels/{run_name}_actin_ground_truth.mrc
-
       is_visualization_default: true
 - metadata:
     annotation_method: 2D CNN + 3D CNN hyperparameter tuning
@@ -99,7 +99,6 @@ annotations:
   - SemanticSegmentationMask:
       file_format: mrc
       glob_string: prediction/{run_name}_*.mrc
-
       is_visualization_default: false
 collection_metadata:
 - sources:

diff --git a/ingestion_tools/dataset_configs/10004.yaml b/ingestion_tools/dataset_configs/10004.yaml
@@ -1,6 +1,7 @@
 alignments:
 - metadata:
     format: IMOD
+    method_type: patch_tracking
   sources:
   - source_multi_glob:
       list_globs:
@@ -69,7 +70,6 @@ annotations:
       columns: xyz
       file_format: csv_with_header
       glob_string: '{run_name}/annoRibosome_bin4.txt'
-
       is_visualization_default: true
 collection_metadata:
 - sources:

diff --git a/ingestion_tools/dataset_configs/10008.yaml b/ingestion_tools/dataset_configs/10008.yaml
@@ -1,6 +1,7 @@
 alignments:
 - metadata:
     format: IMOD
+    method_type: fiducial_based
   sources:
   - source_multi_glob:
       list_globs:
@@ -54,7 +55,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: subtomo/hdcr/hdcr_tm_bin2_motl_12.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -76,7 +76,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: subtomo/ribo/ribo_tm_bin1_motl_12.star
-
       is_visualization_default: true
       order: xyz
 collection_metadata:

diff --git a/ingestion_tools/dataset_configs/10009.yaml b/ingestion_tools/dataset_configs/10009.yaml
@@ -1,6 +1,7 @@
 alignments:
 - metadata:
     format: IMOD
+    method_type: fiducial_based
   sources:
   - source_multi_glob:
       list_globs:
@@ -64,7 +65,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_S_newbin2_3.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -86,7 +86,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_Y_newbin2_3.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -108,7 +107,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_H_cleanbin2.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -131,7 +129,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/MTD_MIP_bin2_recentered_shiftedorg_9.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -153,7 +150,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_B2_newbin2_3.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -175,7 +171,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_A2_newbin2_3.star
-
       is_visualization_default: true
       order: xyz
 - metadata:
@@ -197,7 +192,6 @@ annotations:
       file_format: stopgap_star
       filter_value: '{run_name}'
       glob_string: motivelists/allmotl_D_newbin2_3.star
-
       is_visualization_default: true
       order: xyz
 collection_metadata: