From c9cc5851eb83dc77e40a373c4f3b1518fda34c45 Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Wed, 6 Sep 2023 12:52:42 -0700 Subject: [PATCH 1/8] Adds MetaflowRunView --- chowda/app.py | 3 ++ chowda/fields.py | 34 ++++-------------- chowda/models.py | 8 ++++- chowda/views.py | 7 +++- .../versions/fd7b62eab884_metaflowrun.py | 35 +++++++++++++++++++ 5 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 migrations/versions/fd7b62eab884_metaflowrun.py diff --git a/chowda/app.py b/chowda/app.py index ecd3e73c..636f62d6 100644 --- a/chowda/app.py +++ b/chowda/app.py @@ -20,6 +20,7 @@ ClamsApp, Collection, MediaFile, + MetaflowRun, Pipeline, SonyCiAsset, User, @@ -31,6 +32,7 @@ CollectionView, DashboardView, MediaFileView, + MetaflowRunView, PipelineView, SonyCiAssetView, UserView, @@ -72,6 +74,7 @@ admin.add_view(ClamsAppView(ClamsApp, icon='fa fa-box')) admin.add_view(PipelineView(Pipeline, icon='fa fa-boxes-stacked')) admin.add_view(UserView(User, icon='fa fa-users')) +admin.add_view(MetaflowRunView(MetaflowRun, icon='fa fa-person-running')) # Mount admin to app diff --git a/chowda/fields.py b/chowda/fields.py index 908cdfff..e39f500f 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -104,20 +104,9 @@ class BatchPercentCompleted(BaseField): label: str = 'Completed %' async def parse_obj(self, request: Request, obj: Any) -> Any: - runs = [ - last_run.source - for last_run in [ - media_file.last_metaflow_run_for_batch(batch_id=obj.id) - for media_file in obj.media_files - ] - if last_run - ] - - finished_runs = [run for run in runs if run.finished_at] - if obj.media_files: - percent_completed = len(finished_runs) / len(obj.media_files) - - return f'{percent_completed:.1%}' + runs = [run.finished for run in obj.metaflow_runs] + if runs: + return f'{runs.count(True) / len(obj.media_files):.1%}' return None @@ -131,18 +120,7 @@ class BatchPercentSuccessful(BaseField): exclude_from_edit: bool = True async def parse_obj(self, request: Request, obj: Any) -> Any: - runs = [ - last_run.source - for last_run in [ - media_file.last_metaflow_run_for_batch(batch_id=obj.id) - for media_file in obj.media_files - ] - if last_run - ] - - successful_runs = [run for run in runs if run.successful] - if obj.media_files: - percent_successful = len(successful_runs) / len(obj.media_files) - - return f'{percent_successful:.1%}' + runs = [run.successful for run in obj.metaflow_runs] + if runs: + return f'{runs.count(True) / len(obj.media_files):.1%}' return None diff --git a/chowda/models.py b/chowda/models.py index e9d6424d..d8d21ccd 100644 --- a/chowda/models.py +++ b/chowda/models.py @@ -3,12 +3,14 @@ SQLModels for DB and validation """ +import datetime import enum +from datetime import datetime from typing import Any, Dict, List, Optional from metaflow import Run, namespace from pydantic import AnyHttpUrl, EmailStr, stricturl -from sqlalchemy import JSON, Column, Enum +from sqlalchemy import JSON, Column, DateTime, Enum from sqlalchemy.dialects import postgresql from sqlmodel import Field, Relationship, SQLModel from starlette.requests import Request @@ -253,6 +255,10 @@ class MetaflowRun(SQLModel, table=True): batch: Optional[Batch] = Relationship(back_populates='metaflow_runs') media_file_id: Optional[str] = Field(default=None, foreign_key='media_files.guid') media_file: Optional[MediaFile] = Relationship(back_populates='metaflow_runs') + finished: bool = Field(default=False) + finished_at: Optional[datetime] = Field(sa_column=DateTime, default=None) + duration: Optional[int] = Field(default=None) + successful: Optional[bool] = Field(default=None) @property def source(self): diff --git a/chowda/views.py b/chowda/views.py index adee9cbf..67208060 100644 --- a/chowda/views.py +++ b/chowda/views.py @@ -178,7 +178,8 @@ class BatchView(BaseModelView): label='GUIDs', exclude_from_detail=True, ), - BatchMediaFilesDisplayField(), + # BatchMediaFilesDisplayField(), + 'metaflow_runs', ] async def validate(self, request: Request, data: Dict[str, Any]): @@ -386,3 +387,7 @@ class SonyCiAssetView(AdminModelView): def can_create(self, request: Request) -> bool: """Sony Ci Assets are ingested from Sony Ci API, not created from the UI.""" return False + + +class MetaflowRunView(AdminModelView): + form_include_pk: ClassVar[bool] = True diff --git a/migrations/versions/fd7b62eab884_metaflowrun.py b/migrations/versions/fd7b62eab884_metaflowrun.py new file mode 100644 index 00000000..0ade69e2 --- /dev/null +++ b/migrations/versions/fd7b62eab884_metaflowrun.py @@ -0,0 +1,35 @@ +"""MetaflowRun + +Revision ID: fd7b62eab884 +Revises: 3ae9e767f652 +Create Date: 2023-09-06 12:51:23.899408 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision = 'fd7b62eab884' +down_revision = '3ae9e767f652' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('metaflow_runs', sa.Column('finished', sa.Boolean(), nullable=False)) + op.add_column('metaflow_runs', sa.Column('finished_at', sa.DateTime(), nullable=True)) + op.add_column('metaflow_runs', sa.Column('duration', sa.Integer(), nullable=True)) + op.add_column('metaflow_runs', sa.Column('successful', sa.Boolean(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('metaflow_runs', 'successful') + op.drop_column('metaflow_runs', 'duration') + op.drop_column('metaflow_runs', 'finished_at') + op.drop_column('metaflow_runs', 'finished') + # ### end Alembic commands ### From ea06d01476abf3dbbbdc31014d36c7d918304f90 Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Wed, 6 Sep 2023 13:12:46 -0700 Subject: [PATCH 2/8] Migrates to BatchMetaflowRunDisplayField --- chowda/fields.py | 43 +++++++------------ chowda/views.py | 5 +-- ...ia_files.html => batch_metaflow_runs.html} | 0 3 files changed, 18 insertions(+), 30 deletions(-) rename templates/displays/{batch_media_files.html => batch_metaflow_runs.html} (100%) diff --git a/chowda/fields.py b/chowda/fields.py index e39f500f..7f8abda7 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -58,40 +58,29 @@ async def parse_obj(self, request: Request, obj: Any) -> Any: @dataclass -class BatchMediaFilesDisplayField(BaseField): - """A field that displays a list of MediaFiles in a batch""" +class BatchMetaflowRunDisplayField(BaseField): + """A field that displays a list of MetaflowRuns in a batch""" - name: str = 'batch_media_files' - display_template: str = 'displays/batch_media_files.html' - label: str = 'Media Files' + name: str = 'batch_metaflow_runs' + display_template: str = 'displays/batch_metaflow_runs.html' + label: str = 'Metaflow Runs' exclude_from_edit: bool = True exclude_from_create: bool = True exclude_from_list: bool = True read_only: bool = True async def parse_obj(self, request: Request, obj: Any) -> Any: - media_file_rows = [] - - for media_file in obj.media_files: - media_file_row = {'guid': media_file.guid} - - # Lookup the real Metaflow Run using the last Run ID - run = media_file.last_metaflow_run_for_batch(batch_id=obj.id) - if run: - media_file_row['run_id'] = run.id - media_file_row[ - 'run_link' - ] = f'https://mario.wgbh-mla.org/{run.pathspec}' - media_file_row['finished_at'] = run.source.finished_at or '' - media_file_row['successful'] = run.source.successful - else: - media_file_row['run_id'] = None - media_file_row['run_link'] = None - media_file_row['finished_at'] = None - media_file_row['successful'] = None - - media_file_rows.append(media_file_row) - return media_file_rows + return [ + { + 'guid': run.media_file_id, + 'run_id': run.id, + 'run_link': f'https://mario.wgbh-mla.org/{run.pathspec}', + 'finished_at': run.finished_at or '', + 'finished': run.finished, + 'successful': run.successful, + } + for run in obj.metaflow_runs + ] @dataclass diff --git a/chowda/views.py b/chowda/views.py index 67208060..e4cb55c0 100644 --- a/chowda/views.py +++ b/chowda/views.py @@ -17,7 +17,7 @@ from chowda.auth.utils import get_user from chowda.db import engine from chowda.fields import ( - BatchMediaFilesDisplayField, + BatchMetaflowRunDisplayField, BatchPercentCompleted, BatchPercentSuccessful, MediaFileCount, @@ -178,8 +178,7 @@ class BatchView(BaseModelView): label='GUIDs', exclude_from_detail=True, ), - # BatchMediaFilesDisplayField(), - 'metaflow_runs', + BatchMetaflowRunDisplayField(), ] async def validate(self, request: Request, data: Dict[str, Any]): diff --git a/templates/displays/batch_media_files.html b/templates/displays/batch_metaflow_runs.html similarity index 100% rename from templates/displays/batch_media_files.html rename to templates/displays/batch_metaflow_runs.html From 48ae95ea85595498bf674489a667c94e6db6c7e2 Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Wed, 6 Sep 2023 14:09:50 -0700 Subject: [PATCH 3/8] Fixes ruff error --- chowda/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chowda/models.py b/chowda/models.py index d8d21ccd..4a6aa997 100644 --- a/chowda/models.py +++ b/chowda/models.py @@ -3,7 +3,6 @@ SQLModels for DB and validation """ -import datetime import enum from datetime import datetime from typing import Any, Dict, List, Optional From 184b017faaf974f62cb9e88ac0b9ae74bc031e58 Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Fri, 8 Sep 2023 09:59:11 -0700 Subject: [PATCH 4/8] Updates display field for MetaflowRun to check unfinished runs --- chowda/fields.py | 40 +++++++++++++++------ templates/displays/batch_metaflow_runs.html | 34 +++++++++--------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/chowda/fields.py b/chowda/fields.py index 7f8abda7..bbe02655 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -70,17 +70,35 @@ class BatchMetaflowRunDisplayField(BaseField): read_only: bool = True async def parse_obj(self, request: Request, obj: Any) -> Any: - return [ - { - 'guid': run.media_file_id, - 'run_id': run.id, - 'run_link': f'https://mario.wgbh-mla.org/{run.pathspec}', - 'finished_at': run.finished_at or '', - 'finished': run.finished, - 'successful': run.successful, - } - for run in obj.metaflow_runs - ] + # Check if any runs are still running + running = [run for run in obj.metaflow_runs if not run.finished] + new_runs = None + if running: + from metaflow import Run, namespace + + # Check status of running runs + namespace(None) + runs = [Run(run.pathspec) for run in running] + finished = [run for run in runs if run.finished] + if finished: + from sqlmodel import Session, select + + from chowda.db import engine + from chowda.models import Batch, MetaflowRun + + with Session(engine) as db: + for run in finished: + r = db.exec( + select(MetaflowRun).where(MetaflowRun.id == run.id) + ).one() + r.finished = True + r.successful = run.successful + db.add(r) + db.commit() + # Refresh the data for the page + new_runs = db.get(Batch, obj.id).metaflow_runs + + return [run.dict() for run in new_runs or obj.metaflow_runs] @dataclass diff --git a/templates/displays/batch_metaflow_runs.html b/templates/displays/batch_metaflow_runs.html index 566b2ff3..e98a57c7 100644 --- a/templates/displays/batch_metaflow_runs.html +++ b/templates/displays/batch_metaflow_runs.html @@ -1,4 +1,8 @@ - +
@@ -10,28 +14,24 @@ {% for media_file_row in data %} - - - + + {% endfor %} -
GUID
- {{media_file_row['guid'] }} + {{media_file_row.media_file_id }} - {% if media_file_row['run_link'] %} - - {{ media_file_row['run_id'] }} + + {{ media_file_row.pathspec }} - {% else %} - None - {% endif %} - - {{ media_file_row['finished_at'] }} - - {{ media_file_row['successful'] }} {{ media_file_row.finished }}{{ media_file_row.successful }}
\ No newline at end of file + From d645594952727e4d109e97b0183b27c57456fcf2 Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Fri, 8 Sep 2023 10:05:31 -0700 Subject: [PATCH 5/8] Adds pretty checkmarks for finished, successful --- templates/displays/batch_metaflow_runs.html | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/templates/displays/batch_metaflow_runs.html b/templates/displays/batch_metaflow_runs.html index e98a57c7..6e28dfc9 100644 --- a/templates/displays/batch_metaflow_runs.html +++ b/templates/displays/batch_metaflow_runs.html @@ -7,7 +7,7 @@ GUID Metaflow Run - Finished At + Finished Successful @@ -30,8 +30,22 @@ - {{ media_file_row.finished }} - {{ media_file_row.successful }} + + + + + + {% endfor %} From bd2514675efb39ccb0926f12f821079a3334ad1d Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Fri, 8 Sep 2023 10:15:28 -0700 Subject: [PATCH 6/8] Adds finished_at field to display --- chowda/fields.py | 1 + templates/displays/batch_metaflow_runs.html | 2 ++ 2 files changed, 3 insertions(+) diff --git a/chowda/fields.py b/chowda/fields.py index bbe02655..4d005c5a 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -93,6 +93,7 @@ async def parse_obj(self, request: Request, obj: Any) -> Any: ).one() r.finished = True r.successful = run.successful + r.finished_at = run.finished_at db.add(r) db.commit() # Refresh the data for the page diff --git a/templates/displays/batch_metaflow_runs.html b/templates/displays/batch_metaflow_runs.html index 6e28dfc9..56c621ec 100644 --- a/templates/displays/batch_metaflow_runs.html +++ b/templates/displays/batch_metaflow_runs.html @@ -7,6 +7,7 @@ GUID Metaflow Run + Finished At Finished Successful @@ -30,6 +31,7 @@ + {{ media_file_row.finished_at }} Date: Fri, 8 Sep 2023 10:40:21 -0700 Subject: [PATCH 7/8] Fixes date serializer --- chowda/fields.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/chowda/fields.py b/chowda/fields.py index 4d005c5a..4bb920b6 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -101,6 +101,19 @@ async def parse_obj(self, request: Request, obj: Any) -> Any: return [run.dict() for run in new_runs or obj.metaflow_runs] + async def serialize_value( + self, request: Request, value: Any, action: RequestAction + ) -> Any: + return [ + { + **run, + 'finished_at': run['finished_at'].isoformat() + if run.get('finished_at') + else None, + } + for run in value + ] + @dataclass class BatchPercentCompleted(BaseField): From 2cf3e580cdc640ad00c6c72ea36dee0409ce6f8b Mon Sep 17 00:00:00 2001 From: Harpo Harbert Date: Fri, 8 Sep 2023 11:21:20 -0700 Subject: [PATCH 8/8] Adds created_at and timezone for finished_at --- chowda/fields.py | 3 ++ chowda/models.py | 8 +++-- ...86_metaflowrun_created_at_with_timezone.py | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py diff --git a/chowda/fields.py b/chowda/fields.py index 4bb920b6..7802257f 100644 --- a/chowda/fields.py +++ b/chowda/fields.py @@ -110,6 +110,9 @@ async def serialize_value( 'finished_at': run['finished_at'].isoformat() if run.get('finished_at') else None, + 'created_at': run['created_at'].isoformat() + if run.get('created_at') + else None, } for run in value ] diff --git a/chowda/models.py b/chowda/models.py index 4a6aa997..6515a545 100644 --- a/chowda/models.py +++ b/chowda/models.py @@ -254,9 +254,13 @@ class MetaflowRun(SQLModel, table=True): batch: Optional[Batch] = Relationship(back_populates='metaflow_runs') media_file_id: Optional[str] = Field(default=None, foreign_key='media_files.guid') media_file: Optional[MediaFile] = Relationship(back_populates='metaflow_runs') + created_at: Optional[datetime] = Field( + sa_column=Column(DateTime(timezone=True), default=datetime.utcnow) + ) finished: bool = Field(default=False) - finished_at: Optional[datetime] = Field(sa_column=DateTime, default=None) - duration: Optional[int] = Field(default=None) + finished_at: Optional[datetime] = Field( + sa_column=Column(DateTime(timezone=True), default=None) + ) successful: Optional[bool] = Field(default=None) @property diff --git a/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py b/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py new file mode 100644 index 00000000..a234ba92 --- /dev/null +++ b/migrations/versions/775fe282e786_metaflowrun_created_at_with_timezone.py @@ -0,0 +1,31 @@ +"""MetaflowRun.created_at with timezone + +Revision ID: 775fe282e786 +Revises: fd7b62eab884 +Create Date: 2023-09-08 11:19:21.339487 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision = '775fe282e786' +down_revision = 'fd7b62eab884' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('metaflow_runs', sa.Column('created_at', sa.DateTime(timezone=True), nullable=True)) + op.drop_column('metaflow_runs', 'duration') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('metaflow_runs', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column('metaflow_runs', 'created_at') + # ### end Alembic commands ###