🏃 MetaflowRuns

chowda/app.py

@@ -20,6 +20,7 @@
     ClamsApp,
     Collection,
     MediaFile,
+    MetaflowRun,
     Pipeline,
     SonyCiAsset,
     User,
@@ -31,6 +32,7 @@
     CollectionView,
     DashboardView,
     MediaFileView,
+    MetaflowRunView,
     PipelineView,
     SonyCiAssetView,
     UserView,
@@ -72,6 +74,7 @@
 admin.add_view(ClamsAppView(ClamsApp, icon='fa fa-box'))
 admin.add_view(PipelineView(Pipeline, icon='fa fa-boxes-stacked'))
 admin.add_view(UserView(User, icon='fa fa-users'))
+admin.add_view(MetaflowRunView(MetaflowRun, icon='fa fa-person-running'))
 
 
 # Mount admin to app

chowda/fields.py

@@ -58,40 +58,64 @@ async def parse_obj(self, request: Request, obj: Any) -> Any:
 
 
 @dataclass
-class BatchMediaFilesDisplayField(BaseField):
-    """A field that displays a list of MediaFiles in a batch"""
+class BatchMetaflowRunDisplayField(BaseField):
+    """A field that displays a list of MetaflowRuns in a batch"""
 
-    name: str = 'batch_media_files'
-    display_template: str = 'displays/batch_media_files.html'
-    label: str = 'Media Files'
+    name: str = 'batch_metaflow_runs'
+    display_template: str = 'displays/batch_metaflow_runs.html'
+    label: str = 'Metaflow Runs'
     exclude_from_edit: bool = True
     exclude_from_create: bool = True
     exclude_from_list: bool = True
     read_only: bool = True
 
     async def parse_obj(self, request: Request, obj: Any) -> Any:
-        media_file_rows = []
-
-        for media_file in obj.media_files:
-            media_file_row = {'guid': media_file.guid}
-
-            # Lookup the real Metaflow Run using the last Run ID
-            run = media_file.last_metaflow_run_for_batch(batch_id=obj.id)
-            if run:
-                media_file_row['run_id'] = run.id
-                media_file_row[
-                    'run_link'
-                ] = f'https://mario.wgbh-mla.org/{run.pathspec}'
-                media_file_row['finished_at'] = run.source.finished_at or ''
-                media_file_row['successful'] = run.source.successful
-            else:
-                media_file_row['run_id'] = None
-                media_file_row['run_link'] = None
-                media_file_row['finished_at'] = None
-                media_file_row['successful'] = None
-
-            media_file_rows.append(media_file_row)
-        return media_file_rows
+        # Check if any runs are still running
+        running = [run for run in obj.metaflow_runs if not run.finished]
+        new_runs = None
+        if running:
+            from metaflow import Run, namespace
+
+            # Check status of running runs
+            namespace(None)
+            runs = [Run(run.pathspec) for run in running]
+            finished = [run for run in runs if run.finished]
+            if finished:
+                from sqlmodel import Session, select
+
+                from chowda.db import engine
+                from chowda.models import Batch, MetaflowRun
+
+                with Session(engine) as db:
+                    for run in finished:
+                        r = db.exec(
+                            select(MetaflowRun).where(MetaflowRun.id == run.id)
+                        ).one()
+                        r.finished = True
+                        r.successful = run.successful
+                        r.finished_at = run.finished_at
+                        db.add(r)
+                    db.commit()
+                    # Refresh the data for the page
+                    new_runs = db.get(Batch, obj.id).metaflow_runs
+
+        return [run.dict() for run in new_runs or obj.metaflow_runs]
+
+    async def serialize_value(
+        self, request: Request, value: Any, action: RequestAction
+    ) -> Any:
+        return [
+            {
+                **run,
+                'finished_at': run['finished_at'].isoformat()
+                if run.get('finished_at')
+                else None,
+                'created_at': run['created_at'].isoformat()
+                if run.get('created_at')
+                else None,
+            }
+            for run in value
+        ]
 
 
 @dataclass
@@ -104,20 +128,9 @@ class BatchPercentCompleted(BaseField):
     label: str = 'Completed %'
 
     async def parse_obj(self, request: Request, obj: Any) -> Any:
-        runs = [
-            last_run.source
-            for last_run in [
-                media_file.last_metaflow_run_for_batch(batch_id=obj.id)
-                for media_file in obj.media_files
-            ]
-            if last_run
-        ]
-
-        finished_runs = [run for run in runs if run.finished_at]
-        if obj.media_files:
-            percent_completed = len(finished_runs) / len(obj.media_files)
-
-            return f'{percent_completed:.1%}'
+        runs = [run.finished for run in obj.metaflow_runs]
+        if runs:
+            return f'{runs.count(True) / len(obj.media_files):.1%}'
         return None
 
 
@@ -131,18 +144,7 @@ class BatchPercentSuccessful(BaseField):
     exclude_from_edit: bool = True
 
     async def parse_obj(self, request: Request, obj: Any) -> Any:
-        runs = [
-            last_run.source
-            for last_run in [
-                media_file.last_metaflow_run_for_batch(batch_id=obj.id)
-                for media_file in obj.media_files
-            ]
-            if last_run
-        ]
-
-        successful_runs = [run for run in runs if run.successful]
-        if obj.media_files:
-            percent_successful = len(successful_runs) / len(obj.media_files)
-
-            return f'{percent_successful:.1%}'
+        runs = [run.successful for run in obj.metaflow_runs]
+        if runs:
+            return f'{runs.count(True) / len(obj.media_files):.1%}'
         return None

chowda/models.py

@@ -4,11 +4,12 @@
 """
 
 import enum
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 from metaflow import Run, namespace
 from pydantic import AnyHttpUrl, EmailStr, stricturl
-from sqlalchemy import JSON, Column, Enum
+from sqlalchemy import JSON, Column, DateTime, Enum
 from sqlalchemy.dialects import postgresql
 from sqlmodel import Field, Relationship, SQLModel
 from starlette.requests import Request
@@ -253,6 +254,14 @@ class MetaflowRun(SQLModel, table=True):
     batch: Optional[Batch] = Relationship(back_populates='metaflow_runs')
     media_file_id: Optional[str] = Field(default=None, foreign_key='media_files.guid')
     media_file: Optional[MediaFile] = Relationship(back_populates='metaflow_runs')
+    created_at: Optional[datetime] = Field(
+        sa_column=Column(DateTime(timezone=True), default=datetime.utcnow)
+    )
+    finished: bool = Field(default=False)
+    finished_at: Optional[datetime] = Field(
+        sa_column=Column(DateTime(timezone=True), default=None)
+    )
+    successful: Optional[bool] = Field(default=None)
 
     @property
     def source(self):

chowda/views.py

@@ -17,7 +17,7 @@
 from chowda.auth.utils import get_user
 from chowda.db import engine
 from chowda.fields import (
-    BatchMediaFilesDisplayField,
+    BatchMetaflowRunDisplayField,
     BatchPercentCompleted,
     BatchPercentSuccessful,
     MediaFileCount,
@@ -178,7 +178,7 @@ class BatchView(BaseModelView):
             label='GUIDs',
             exclude_from_detail=True,
         ),
-        BatchMediaFilesDisplayField(),
+        BatchMetaflowRunDisplayField(),
     ]
 
     async def validate(self, request: Request, data: Dict[str, Any]):
@@ -386,3 +386,7 @@ class SonyCiAssetView(AdminModelView):
     def can_create(self, request: Request) -> bool:
         """Sony Ci Assets are ingested from Sony Ci API, not created from the UI."""
         return False
+
+
+class MetaflowRunView(AdminModelView):
+    form_include_pk: ClassVar[bool] = True

migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py

@@ -0,0 +1,31 @@
+"""MetaflowRun.created_at with timezone
+
+Revision ID: 775fe282e786
+Revises: fd7b62eab884
+Create Date: 2023-09-08 11:19:21.339487
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision = '775fe282e786'
+down_revision = 'fd7b62eab884'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('metaflow_runs', sa.Column('created_at', sa.DateTime(timezone=True), nullable=True))
+    op.drop_column('metaflow_runs', 'duration')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('metaflow_runs', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.drop_column('metaflow_runs', 'created_at')
+    # ### end Alembic commands ###

migrations/versions/fd7b62eab884_metaflowrun.py

@@ -0,0 +1,35 @@
+"""MetaflowRun
+
+Revision ID: fd7b62eab884
+Revises: 3ae9e767f652
+Create Date: 2023-09-06 12:51:23.899408
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision = 'fd7b62eab884'
+down_revision = '3ae9e767f652'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('metaflow_runs', sa.Column('finished', sa.Boolean(), nullable=False))
+    op.add_column('metaflow_runs', sa.Column('finished_at', sa.DateTime(), nullable=True))
+    op.add_column('metaflow_runs', sa.Column('duration', sa.Integer(), nullable=True))
+    op.add_column('metaflow_runs', sa.Column('successful', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('metaflow_runs', 'successful')
+    op.drop_column('metaflow_runs', 'duration')
+    op.drop_column('metaflow_runs', 'finished_at')
+    op.drop_column('metaflow_runs', 'finished')
+    # ### end Alembic commands ###

templates/displays/batch_metaflow_runs.html

@@ -0,0 +1,53 @@
+<table
+  id="dt"
+  class="table table-vcenter text-nowrap dataTable no-footer"
+  aria-describedby="dt_info"
+>
+  <thead>
+    <tr>
+      <th>GUID</th>
+      <th>Metaflow Run</th>
+      <th>Finished At</th>
+      <th>Finished</th>
+      <th>Successful</th>
+    </tr>
+  </thead>
+
+  {% for media_file_row in data %}
+  <tr>
+    <td>
+      <a
+        class="m-1 py-1 px-2 badge bg-blue-lt lead"
+        href="../../media-file/detail/{{ media_file_row.media_file_id }}"
+        >{{media_file_row.media_file_id }}
+      </a>
+    </td>
+    <td>
+      <a
+        href="https://mario.wgbh-mla.org/{{ media_file_row.pathspec}}"
+        target="blank"
+      >
+        {{ media_file_row.pathspec }}
+        <i class="fa fa-external-link" aria-hidden="true"></i>
+      </a>
+    </td>
+    <td>{{ media_file_row.finished_at }}</td>
+    <td>
+      <span
+        class="text-center text-{{'success' if media_file_row.finished else 'danger'}}"
+        ><i
+          class="fa-solid fa-{{'check' if media_file_row.finished else 'times'}}-circle fa-lg"
+        ></i
+      ></span>
+    </td>
+    <td>
+      <span
+        class="text-center text-{{'success' if media_file_row.successful else 'danger'}}"
+        ><i
+          class="fa-solid fa-{{'check' if media_file_row.successful else 'times'}}-circle fa-lg"
+        ></i
+      ></span>
+    </td>
+  </tr>
+  {% endfor %}
+</table>