diff --git a/chowda/app.py b/chowda/app.py
index ecd3e73c..636f62d6 100644
--- a/chowda/app.py
+++ b/chowda/app.py
@@ -20,6 +20,7 @@
ClamsApp,
Collection,
MediaFile,
+ MetaflowRun,
Pipeline,
SonyCiAsset,
User,
@@ -31,6 +32,7 @@
CollectionView,
DashboardView,
MediaFileView,
+ MetaflowRunView,
PipelineView,
SonyCiAssetView,
UserView,
@@ -72,6 +74,7 @@
admin.add_view(ClamsAppView(ClamsApp, icon='fa fa-box'))
admin.add_view(PipelineView(Pipeline, icon='fa fa-boxes-stacked'))
admin.add_view(UserView(User, icon='fa fa-users'))
+admin.add_view(MetaflowRunView(MetaflowRun, icon='fa fa-person-running'))
# Mount admin to app
diff --git a/chowda/fields.py b/chowda/fields.py
index 908cdfff..7802257f 100644
--- a/chowda/fields.py
+++ b/chowda/fields.py
@@ -58,40 +58,64 @@ async def parse_obj(self, request: Request, obj: Any) -> Any:
@dataclass
-class BatchMediaFilesDisplayField(BaseField):
- """A field that displays a list of MediaFiles in a batch"""
+class BatchMetaflowRunDisplayField(BaseField):
+ """A field that displays a list of MetaflowRuns in a batch"""
- name: str = 'batch_media_files'
- display_template: str = 'displays/batch_media_files.html'
- label: str = 'Media Files'
+ name: str = 'batch_metaflow_runs'
+ display_template: str = 'displays/batch_metaflow_runs.html'
+ label: str = 'Metaflow Runs'
exclude_from_edit: bool = True
exclude_from_create: bool = True
exclude_from_list: bool = True
read_only: bool = True
async def parse_obj(self, request: Request, obj: Any) -> Any:
- media_file_rows = []
-
- for media_file in obj.media_files:
- media_file_row = {'guid': media_file.guid}
-
- # Lookup the real Metaflow Run using the last Run ID
- run = media_file.last_metaflow_run_for_batch(batch_id=obj.id)
- if run:
- media_file_row['run_id'] = run.id
- media_file_row[
- 'run_link'
- ] = f'https://mario.wgbh-mla.org/{run.pathspec}'
- media_file_row['finished_at'] = run.source.finished_at or ''
- media_file_row['successful'] = run.source.successful
- else:
- media_file_row['run_id'] = None
- media_file_row['run_link'] = None
- media_file_row['finished_at'] = None
- media_file_row['successful'] = None
-
- media_file_rows.append(media_file_row)
- return media_file_rows
+ # Check if any runs are still running
+ running = [run for run in obj.metaflow_runs if not run.finished]
+ new_runs = None
+ if running:
+ from metaflow import Run, namespace
+
+ # Check status of running runs
+ namespace(None)
+ runs = [Run(run.pathspec) for run in running]
+ finished = [run for run in runs if run.finished]
+ if finished:
+ from sqlmodel import Session, select
+
+ from chowda.db import engine
+ from chowda.models import Batch, MetaflowRun
+
+ with Session(engine) as db:
+ for run in finished:
+ r = db.exec(
+ select(MetaflowRun).where(MetaflowRun.id == run.id)
+ ).one()
+ r.finished = True
+ r.successful = run.successful
+ r.finished_at = run.finished_at
+ db.add(r)
+ db.commit()
+ # Refresh the data for the page
+ new_runs = db.get(Batch, obj.id).metaflow_runs
+
+ return [run.dict() for run in new_runs or obj.metaflow_runs]
+
+ async def serialize_value(
+ self, request: Request, value: Any, action: RequestAction
+ ) -> Any:
+ return [
+ {
+ **run,
+ 'finished_at': run['finished_at'].isoformat()
+ if run.get('finished_at')
+ else None,
+ 'created_at': run['created_at'].isoformat()
+ if run.get('created_at')
+ else None,
+ }
+ for run in value
+ ]
@dataclass
@@ -104,20 +128,9 @@ class BatchPercentCompleted(BaseField):
label: str = 'Completed %'
async def parse_obj(self, request: Request, obj: Any) -> Any:
- runs = [
- last_run.source
- for last_run in [
- media_file.last_metaflow_run_for_batch(batch_id=obj.id)
- for media_file in obj.media_files
- ]
- if last_run
- ]
-
- finished_runs = [run for run in runs if run.finished_at]
- if obj.media_files:
- percent_completed = len(finished_runs) / len(obj.media_files)
-
- return f'{percent_completed:.1%}'
+ runs = [run.finished for run in obj.metaflow_runs]
+ if runs:
+ return f'{runs.count(True) / len(obj.media_files):.1%}'
return None
@@ -131,18 +144,7 @@ class BatchPercentSuccessful(BaseField):
exclude_from_edit: bool = True
async def parse_obj(self, request: Request, obj: Any) -> Any:
- runs = [
- last_run.source
- for last_run in [
- media_file.last_metaflow_run_for_batch(batch_id=obj.id)
- for media_file in obj.media_files
- ]
- if last_run
- ]
-
- successful_runs = [run for run in runs if run.successful]
- if obj.media_files:
- percent_successful = len(successful_runs) / len(obj.media_files)
-
- return f'{percent_successful:.1%}'
+ runs = [run.successful for run in obj.metaflow_runs]
+ if runs:
+ return f'{runs.count(True) / len(obj.media_files):.1%}'
return None
diff --git a/chowda/models.py b/chowda/models.py
index e9d6424d..6515a545 100644
--- a/chowda/models.py
+++ b/chowda/models.py
@@ -4,11 +4,12 @@
"""
import enum
+from datetime import datetime
from typing import Any, Dict, List, Optional
from metaflow import Run, namespace
from pydantic import AnyHttpUrl, EmailStr, stricturl
-from sqlalchemy import JSON, Column, Enum
+from sqlalchemy import JSON, Column, DateTime, Enum
from sqlalchemy.dialects import postgresql
from sqlmodel import Field, Relationship, SQLModel
from starlette.requests import Request
@@ -253,6 +254,14 @@ class MetaflowRun(SQLModel, table=True):
batch: Optional[Batch] = Relationship(back_populates='metaflow_runs')
media_file_id: Optional[str] = Field(default=None, foreign_key='media_files.guid')
media_file: Optional[MediaFile] = Relationship(back_populates='metaflow_runs')
+ created_at: Optional[datetime] = Field(
+ sa_column=Column(DateTime(timezone=True), default=datetime.utcnow)
+ )
+ finished: bool = Field(default=False)
+ finished_at: Optional[datetime] = Field(
+ sa_column=Column(DateTime(timezone=True), default=None)
+ )
+ successful: Optional[bool] = Field(default=None)
@property
def source(self):
diff --git a/chowda/views.py b/chowda/views.py
index adee9cbf..e4cb55c0 100644
--- a/chowda/views.py
+++ b/chowda/views.py
@@ -17,7 +17,7 @@
from chowda.auth.utils import get_user
from chowda.db import engine
from chowda.fields import (
- BatchMediaFilesDisplayField,
+ BatchMetaflowRunDisplayField,
BatchPercentCompleted,
BatchPercentSuccessful,
MediaFileCount,
@@ -178,7 +178,7 @@ class BatchView(BaseModelView):
label='GUIDs',
exclude_from_detail=True,
),
- BatchMediaFilesDisplayField(),
+ BatchMetaflowRunDisplayField(),
]
async def validate(self, request: Request, data: Dict[str, Any]):
@@ -386,3 +386,7 @@ class SonyCiAssetView(AdminModelView):
def can_create(self, request: Request) -> bool:
"""Sony Ci Assets are ingested from Sony Ci API, not created from the UI."""
return False
+
+
+class MetaflowRunView(AdminModelView):
+ form_include_pk: ClassVar[bool] = True
diff --git a/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py b/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py
new file mode 100644
index 00000000..a234ba92
--- /dev/null
+++ b/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py
@@ -0,0 +1,31 @@
+"""MetaflowRun.created_at with timezone
+
+Revision ID: 775fe282e786
+Revises: fd7b62eab884
+Create Date: 2023-09-08 11:19:21.339487
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision = '775fe282e786'
+down_revision = 'fd7b62eab884'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('metaflow_runs', sa.Column('created_at', sa.DateTime(timezone=True), nullable=True))
+ op.drop_column('metaflow_runs', 'duration')
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('metaflow_runs', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True))
+ op.drop_column('metaflow_runs', 'created_at')
+ # ### end Alembic commands ###
diff --git a/migrations/versions/fd7b62eab884_metaflowrun.py b/migrations/versions/fd7b62eab884_metaflowrun.py
new file mode 100644
index 00000000..0ade69e2
--- /dev/null
+++ b/migrations/versions/fd7b62eab884_metaflowrun.py
@@ -0,0 +1,35 @@
+"""MetaflowRun
+
+Revision ID: fd7b62eab884
+Revises: 3ae9e767f652
+Create Date: 2023-09-06 12:51:23.899408
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision = 'fd7b62eab884'
+down_revision = '3ae9e767f652'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column('metaflow_runs', sa.Column('finished', sa.Boolean(), nullable=False))
+ op.add_column('metaflow_runs', sa.Column('finished_at', sa.DateTime(), nullable=True))
+ op.add_column('metaflow_runs', sa.Column('duration', sa.Integer(), nullable=True))
+ op.add_column('metaflow_runs', sa.Column('successful', sa.Boolean(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column('metaflow_runs', 'successful')
+ op.drop_column('metaflow_runs', 'duration')
+ op.drop_column('metaflow_runs', 'finished_at')
+ op.drop_column('metaflow_runs', 'finished')
+ # ### end Alembic commands ###
diff --git a/templates/displays/batch_media_files.html b/templates/displays/batch_media_files.html
deleted file mode 100644
index 566b2ff3..00000000
--- a/templates/displays/batch_media_files.html
+++ /dev/null
@@ -1,37 +0,0 @@
-
\ No newline at end of file
diff --git a/templates/displays/batch_metaflow_runs.html b/templates/displays/batch_metaflow_runs.html
new file mode 100644
index 00000000..56c621ec
--- /dev/null
+++ b/templates/displays/batch_metaflow_runs.html
@@ -0,0 +1,53 @@
+