Merge pull request #215 from supervisely-ecosystem/workflow

Add Workflow and Versioning

dev_requirements.txt

@@ -1,7 +1,7 @@
 # git+https://github.com/supervisely/supervisely.git@test-branch
 
 # pip install -r requirements.txt
-supervisely==6.73.122
+supervisely==6.73.123
 
 opencv-python-headless==4.5.5.62
 opencv-python==4.5.5.62

supervisely/serve/config.json

@@ -11,7 +11,7 @@
     "serve"
   ],
   "description": "Deploy model as REST API service",
-  "docker_image": "supervisely/yolov5:1.0.7",
+  "docker_image": "supervisely/yolov5:1.0.8",
   "instance_version": "6.8.88",
   "entrypoint": "python -m uvicorn main:m.app --app-dir ./supervisely/serve/src --host 0.0.0.0 --port 8000 --ws websockets",
   "port": 8000,

supervisely/serve/src/main.py

@@ -18,11 +18,14 @@
 from utils.datasets import letterbox
 from pathlib import Path
 
+
 root_source_path = str(Path(__file__).parents[3])
 app_source_path = str(Path(__file__).parents[1])
 load_dotenv(os.path.join(app_source_path, "local.env"))
 load_dotenv(os.path.expanduser("~/supervisely.env"))
 
+from workflow import Workflow
+
 model_weights_options = os.environ["modal.state.modelWeightsOptions"]
 pretrained_weights = os.environ["modal.state.selectedModel"].lower()
 custom_weights = os.environ["modal.state.weightsPath"]
@@ -45,6 +48,8 @@ def load_on_device(
             self.local_weights_path = self.download(custom_weights)
             cfg_path_in_teamfiles = os.path.join(Path(custom_weights).parents[1], "opt.yaml")
             configs_local_path = self.download(cfg_path_in_teamfiles)
+            workflow = Workflow(self.api)
+            workflow.add_input(custom_weights)
 
         self.device = select_device(device)
         self.half = self.device.type != "cpu"  # half precision only supported on CUDA

supervisely/serve/src/workflow.py

@@ -0,0 +1,49 @@
+import supervisely as sly
+
+
+def check_compatibility(func):
+    def wrapper(self, *args, **kwargs):
+        if self.is_compatible is None:
+            self.is_compatible = self.check_instance_ver_compatibility()
+        if not self.is_compatible:
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class Workflow:
+    def __init__(self, api: sly.Api, min_instance_version: str = None):
+        self.is_compatible = None
+        self.api = api
+        self._min_instance_version = (
+            "6.9.31" if min_instance_version is None else min_instance_version
+        )
+
+    def check_instance_ver_compatibility(self):
+        if not self.api.is_version_supported(self._min_instance_version):
+            sly.logger.info(
+                f"Supervisely instance version {self.api.instance_version} does not support workflow features."
+            )
+            if not sly.is_community():
+                sly.logger.info(
+                    f"To use them, please update your instance to version {self._min_instance_version} or higher."
+                )
+            return False
+        return True
+
+    @check_compatibility
+    def add_input(self, checkpoint_url: str):
+        try:
+            meta = {"customNodeSettings": {"title": "<h4>Serve Custom Model</h4>"}}
+            sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}")
+            if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url):
+                self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta)
+            else:
+                sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input")
+        except Exception as e:
+            sly.logger.error(f"Failed to add input to the workflow: {e}")
+
+    @check_compatibility
+    def add_output(self):
+        raise NotImplementedError("add_output is not implemented in this workflow")

supervisely/train/config.json

@@ -10,7 +10,7 @@
     "train"
   ],
   "description": "Dashboard to configure and monitor training",
-  "docker_image": "supervisely/yolov5:1.0.7",
+  "docker_image": "supervisely/yolov5:1.0.8",
   "min_instance_version": "6.8.70",
   "main_script": "supervisely/train/src/sly_train.py",
   "gui_template": "supervisely/train/src/gui.html",

supervisely/train/src/sly_train.py

@@ -40,6 +40,10 @@ def train(api: sly.Api, task_id, context, state, app_logger):
         project_dir = os.path.join(my_app.data_dir, "sly_project")
         sly.fs.mkdir(project_dir, remove_content_if_exists=True)  # clean content for debug, has no effect in prod
 
+        # -------------------------------------- Add Workflow Input -------------------------------------- #
+        g.workflow.add_input(g.project_info, state)
+        # ----------------------------------------------- - ---------------------------------------------- #
+
         # download and preprocess Sypervisely project (using cache)
         try:
             download_project(
@@ -145,6 +149,7 @@ def train(api: sly.Api, task_id, context, state, app_logger):
         # upload artifacts directory to Team Files
         upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir)
         set_task_output()
+        g.workflow.add_output(state, g.remote_artifacts_dir)
     except Exception as e:
         msg = f"Something went wrong. Find more info in the app logs."
         my_app.show_modal_window(f"{msg} {repr(e)}", level="error", log_message=False)

supervisely/train/src/sly_train_globals.py

@@ -6,6 +6,7 @@
 from supervisely.nn.artifacts.yolov5 import YOLOv5
 from supervisely.app.v1.app_service import AppService
 from dotenv import load_dotenv
+from workflow import Workflow
 
 root_source_dir = str(Path(sys.argv[0]).parents[3])
 sly.logger.info(f"Root source directory: {root_source_dir}")
@@ -35,6 +36,7 @@
 project_id = int(os.environ['modal.state.slyProjectId'])
 
 api: sly.Api = my_app.public_api
+workflow = Workflow(api)
 task_id = my_app.task_id
 
 local_artifacts_dir = None

supervisely/train/src/workflow.py

@@ -0,0 +1,101 @@
+# Description: This file contains versioning features and the Workflow class that is used to add input and output to the workflow.
+
+import supervisely as sly
+import os
+
+
+def check_compatibility(func):
+    def wrapper(self, *args, **kwargs):
+        if self.is_compatible is None:
+            self.is_compatible = self.check_instance_ver_compatibility()
+        if not self.is_compatible:
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class Workflow:
+    def __init__(self, api: sly.Api, min_instance_version: str = None):
+        self.is_compatible = None
+        self.api = api
+        self._min_instance_version = (
+            "6.9.31" if min_instance_version is None else min_instance_version
+        )
+
+    def check_instance_ver_compatibility(self):
+        if not self.api.is_version_supported(self._min_instance_version):
+            sly.logger.info(
+                f"Supervisely instance version {self.api.instance_version} does not support workflow and versioning features."
+            )
+            if not sly.is_community():
+                sly.logger.info(
+                    f"To use them, please update your instance to version {self._min_instance_version} or higher."
+                )
+            return False
+        return True
+
+    @check_compatibility
+    def add_input(self, project_info: sly.ProjectInfo, state: dict):
+        try:
+            project_version_id = self.api.project.version.create(
+                project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}"
+            )
+        except Exception as e:
+            sly.logger.error(f"Failed to create a project version: {e}")
+            project_version_id = None
+
+        try:
+            if project_version_id is None:
+                project_version_id = project_info.version.get("id", None) if project_info.version else None
+            self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id)
+            file_info = False
+            if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom":
+                file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"])
+                self.api.app.workflow.add_input_file(file_info, model_weight=True)
+            sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}")
+        except Exception as e:
+            sly.logger.error(f"Failed to add input to the workflow: {e}")
+
+    @check_compatibility
+    def add_output(self, state: dict, team_files_dir: str):
+        try:
+            weights_dir_in_team_files = os.path.join(team_files_dir, "weights")
+            files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo")
+            best_filename_info = None
+            for file_info in files_info:
+                if "best" in file_info.name:
+                    best_filename_info = file_info
+                    break
+            if best_filename_info:
+                module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id")                       
+                if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom":
+                    model_name = "Custom Model"
+                else:
+                    model_name = "YOLOv5"
+
+                meta = {
+                    "customNodeSettings": {
+                    "title": f"<h4>Train {model_name}</h4>",
+                    "mainLink": {
+                        "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}",
+                        "title": "Show Results"
+                    }
+                },
+                "customRelationSettings": {
+                    "icon": {
+                        "icon": "zmdi-folder",
+                        "color": "#FFA500",
+                        "backgroundColor": "#FFE8BE"
+                    },
+                    "title": "<h4>Checkpoints</h4>",
+                    "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"}
+                    }
+                }
+                sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}")
+                sly.logger.debug(f"Workflow Output: meta \n    {meta}")
+                self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta)
+            else:
+                sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.")
+        except Exception as e:
+            sly.logger.error(f"Failed to add output to the workflow: {e}")