From 1de3ac945bffa9578dd424caa8593bff96417128 Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 6 Jun 2024 17:04:00 +0400 Subject: [PATCH 1/8] update train --- supervisely/train/src/sly_train_globals.py | 11 ++++++++++- supervisely/train/src/sly_utils.py | 13 ++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 84b6b054246d..4c2547b71ea2 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -3,6 +3,7 @@ import sys import yaml import supervisely as sly +from supervisely.nn.checkpoints.yolov5 import YOLOv5Checkpoint from supervisely.app.v1.app_service import AppService from dotenv import load_dotenv @@ -54,6 +55,14 @@ experiment_name = str(task_id) local_artifacts_dir = os.path.join(runs_dir, experiment_name) sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") -remote_artifacts_dir = os.path.join("/yolov5_train", project_info.name, experiment_name) + +checkpoint = YOLOv5Checkpoint(team_id) +model_dir = checkpoint.get_model_dir() + +remote_artifacts_dir = os.path.join(model_dir, project_info.name, experiment_name) remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) + +remote_weights_dir = os.path.join(remote_artifacts_dir, checkpoint.weights_dir) +remote_weights_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) + sly.logger.info(f"After training artifacts will be uploaded to Team Files: {remote_artifacts_dir}") diff --git a/supervisely/train/src/sly_utils.py b/supervisely/train/src/sly_utils.py index bba1c326b95d..ad6101afc029 100644 --- a/supervisely/train/src/sly_utils.py +++ b/supervisely/train/src/sly_utils.py @@ -66,4 +66,15 @@ def _gen_message(current, total): globals.api.file.upload(globals.team_id, local_path, remote_path, lambda monitor: progress_cb(progress_last + monitor.bytes_read)) progress.message = _gen_message(idx + 1, len(local_files)) - time.sleep(0.5) \ No newline at end of file + time.sleep(0.5) + + # generate metadata + globals.checkpoint.generate_sly_metadata( + app_name=globals.checkpoint.app_name, + session_id=globals.experiment_name, + session_path=globals.remote_artifacts_dir, + weights_dir=globals.remote_weights_dir, + training_project_name=globals.project_info.name, + task_type=globals.checkpoint.task_type, + config_path=None, + ) \ No newline at end of file From 8c9a14c186007d73de62aacd485479150cdc8958 Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 6 Jun 2024 17:06:39 +0400 Subject: [PATCH 2/8] add reqs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100755 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 000000000000..f98cd10d25a4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/supervisely/supervisely.git@optimize-checkpoint From 0094d8906133ae91c00d4ca0d8a125637d77bb93 Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 6 Jun 2024 18:02:10 +0400 Subject: [PATCH 3/8] fix typo --- supervisely/train/src/sly_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisely/train/src/sly_utils.py b/supervisely/train/src/sly_utils.py index ad6101afc029..565b9b6f230d 100644 --- a/supervisely/train/src/sly_utils.py +++ b/supervisely/train/src/sly_utils.py @@ -73,7 +73,7 @@ def _gen_message(current, total): app_name=globals.checkpoint.app_name, session_id=globals.experiment_name, session_path=globals.remote_artifacts_dir, - weights_dir=globals.remote_weights_dir, + weights_path=globals.remote_weights_dir, training_project_name=globals.project_info.name, task_type=globals.checkpoint.task_type, config_path=None, From 69ccc4d584ebfb53136714ade31d2f1a89cc120a Mon Sep 17 00:00:00 2001 From: cxnt Date: Fri, 7 Jun 2024 15:43:22 +0400 Subject: [PATCH 4/8] update train --- supervisely/train/src/sly_train_globals.py | 8 ++++---- supervisely/train/src/sly_utils.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 4c2547b71ea2..f3249c2e03cf 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -3,7 +3,7 @@ import sys import yaml import supervisely as sly -from supervisely.nn.checkpoints.yolov5 import YOLOv5Checkpoint +from supervisely.nn.models.yolov5 import YOLOv5 from supervisely.app.v1.app_service import AppService from dotenv import load_dotenv @@ -56,13 +56,13 @@ local_artifacts_dir = os.path.join(runs_dir, experiment_name) sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") -checkpoint = YOLOv5Checkpoint(team_id) -model_dir = checkpoint.get_model_dir() +sly_yolov5 = YOLOv5(team_id) +model_dir = sly_yolov5.framework_dir remote_artifacts_dir = os.path.join(model_dir, project_info.name, experiment_name) remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) -remote_weights_dir = os.path.join(remote_artifacts_dir, checkpoint.weights_dir) +remote_weights_dir = os.path.join(remote_artifacts_dir, sly_yolov5.weights_dir) remote_weights_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) sly.logger.info(f"After training artifacts will be uploaded to Team Files: {remote_artifacts_dir}") diff --git a/supervisely/train/src/sly_utils.py b/supervisely/train/src/sly_utils.py index 565b9b6f230d..e8b761725243 100644 --- a/supervisely/train/src/sly_utils.py +++ b/supervisely/train/src/sly_utils.py @@ -69,12 +69,12 @@ def _gen_message(current, total): time.sleep(0.5) # generate metadata - globals.checkpoint.generate_sly_metadata( - app_name=globals.checkpoint.app_name, + globals.sly_yolov5.generate_sly_metadata( + app_name=globals.sly_yolov5.app_name, session_id=globals.experiment_name, session_path=globals.remote_artifacts_dir, weights_path=globals.remote_weights_dir, training_project_name=globals.project_info.name, - task_type=globals.checkpoint.task_type, + task_type=globals.sly_yolov5.task_type, config_path=None, ) \ No newline at end of file From 709f939739274d6ac3a0e1e770505176064205d3 Mon Sep 17 00:00:00 2001 From: cxnt Date: Fri, 7 Jun 2024 19:30:41 +0400 Subject: [PATCH 5/8] update train --- supervisely/train/src/sly_train_globals.py | 8 ++++---- supervisely/train/src/sly_utils.py | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index f3249c2e03cf..6a5e553d6839 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -3,7 +3,7 @@ import sys import yaml import supervisely as sly -from supervisely.nn.models.yolov5 import YOLOv5 +from supervisely.nn.artifacts.yolov5 import YOLOv5 from supervisely.app.v1.app_service import AppService from dotenv import load_dotenv @@ -57,12 +57,12 @@ sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") sly_yolov5 = YOLOv5(team_id) -model_dir = sly_yolov5.framework_dir +framework_dir = sly_yolov5.framework_dir -remote_artifacts_dir = os.path.join(model_dir, project_info.name, experiment_name) +remote_artifacts_dir = os.path.join(framework_dir, project_info.name, experiment_name) remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) -remote_weights_dir = os.path.join(remote_artifacts_dir, sly_yolov5.weights_dir) +remote_weights_dir = sly_yolov5.get_weights_path(remote_artifacts_dir) remote_weights_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) sly.logger.info(f"After training artifacts will be uploaded to Team Files: {remote_artifacts_dir}") diff --git a/supervisely/train/src/sly_utils.py b/supervisely/train/src/sly_utils.py index e8b761725243..f5fae2e02464 100644 --- a/supervisely/train/src/sly_utils.py +++ b/supervisely/train/src/sly_utils.py @@ -69,12 +69,13 @@ def _gen_message(current, total): time.sleep(0.5) # generate metadata - globals.sly_yolov5.generate_sly_metadata( + globals.sly_yolov5.generate_metadata( app_name=globals.sly_yolov5.app_name, - session_id=globals.experiment_name, - session_path=globals.remote_artifacts_dir, - weights_path=globals.remote_weights_dir, - training_project_name=globals.project_info.name, + task_id=globals.experiment_name, + artifacts_folder=globals.remote_artifacts_dir, + weights_folder=globals.remote_weights_dir, + weights_ext=globals.sly_yolov5.weights_ext, + project_name=globals.project_info.name, task_type=globals.sly_yolov5.task_type, config_path=None, ) \ No newline at end of file From cd36fb43d76fc23e54289c9cbaf232698a9f7e8c Mon Sep 17 00:00:00 2001 From: cxnt Date: Fri, 7 Jun 2024 19:37:28 +0400 Subject: [PATCH 6/8] update train --- supervisely/train/src/sly_train_globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 6a5e553d6839..a2245c09dc57 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -57,7 +57,7 @@ sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") sly_yolov5 = YOLOv5(team_id) -framework_dir = sly_yolov5.framework_dir +framework_dir = sly_yolov5._framework_folder remote_artifacts_dir = os.path.join(framework_dir, project_info.name, experiment_name) remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) From 0303a950ae18830b98cf90457b25e31fdb0e7a08 Mon Sep 17 00:00:00 2001 From: cxnt Date: Mon, 10 Jun 2024 12:50:00 +0400 Subject: [PATCH 7/8] update build action --- ..._push_image_manual.yml => build_image.yml} | 23 +++++++++---- .../manual-build-push-docker-image.yml | 33 ------------------- dev_requirements.txt | 4 ++- requirements.txt | 1 - supervisely/train/src/sly_train_globals.py | 2 +- supervisely/train/src/sly_utils.py | 2 +- 6 files changed, 22 insertions(+), 43 deletions(-) rename .github/workflows/{sly_build_and_push_image_manual.yml => build_image.yml} (50%) delete mode 100644 .github/workflows/manual-build-push-docker-image.yml delete mode 100755 requirements.txt diff --git a/.github/workflows/sly_build_and_push_image_manual.yml b/.github/workflows/build_image.yml similarity index 50% rename from .github/workflows/sly_build_and_push_image_manual.yml rename to .github/workflows/build_image.yml index 24de3c7f002c..418043aaf28c 100644 --- a/.github/workflows/sly_build_and_push_image_manual.yml +++ b/.github/workflows/build_image.yml @@ -1,10 +1,10 @@ -name: Supervisely Build and Push Image Manual +name: Docker Image Build on: workflow_dispatch: inputs: tag_version: - description: 'Docker Image Tag' + description: 'Docker Image Tag (without "v")' required: true default: '' @@ -12,21 +12,32 @@ jobs: docker: runs-on: ubuntu-latest steps: + - + name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: true - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: push: true file: Dockerfile diff --git a/.github/workflows/manual-build-push-docker-image.yml b/.github/workflows/manual-build-push-docker-image.yml deleted file mode 100644 index 9996dc6a942f..000000000000 --- a/.github/workflows/manual-build-push-docker-image.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Dokcer Image CI - -on: - workflow_dispatch: - inputs: - tag_version: - description: 'Docker Image Tag' - required: true - default: '' - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v3 - with: - push: true - file: Dockerfile - tags: supervisely/yolov5:${{ github.event.inputs.tag_version }} diff --git a/dev_requirements.txt b/dev_requirements.txt index 0b28dd464de6..e97ca50a43cd 100755 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,7 @@ +# git+https://github.com/supervisely/supervisely.git@test-branch + # pip install -r requirements.txt -supervisely==6.73.95 +supervisely==6.73.100 opencv-python-headless==4.5.5.62 opencv-python==4.5.5.62 diff --git a/requirements.txt b/requirements.txt deleted file mode 100755 index f98cd10d25a4..000000000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://github.com/supervisely/supervisely.git@optimize-checkpoint diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index a2245c09dc57..3e0e1d16d6e3 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -57,7 +57,7 @@ sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") sly_yolov5 = YOLOv5(team_id) -framework_dir = sly_yolov5._framework_folder +framework_dir = sly_yolov5.framework_folder remote_artifacts_dir = os.path.join(framework_dir, project_info.name, experiment_name) remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) diff --git a/supervisely/train/src/sly_utils.py b/supervisely/train/src/sly_utils.py index f5fae2e02464..17dcc2140076 100644 --- a/supervisely/train/src/sly_utils.py +++ b/supervisely/train/src/sly_utils.py @@ -78,4 +78,4 @@ def _gen_message(current, total): project_name=globals.project_info.name, task_type=globals.sly_yolov5.task_type, config_path=None, - ) \ No newline at end of file + ) From bfb8af24beb464285f2e2f502c9d1e4f0229b1f3 Mon Sep 17 00:00:00 2001 From: cxnt Date: Mon, 10 Jun 2024 14:25:18 +0400 Subject: [PATCH 8/8] update docker --- supervisely/serve/config.json | 2 +- supervisely/train/config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/supervisely/serve/config.json b/supervisely/serve/config.json index 9c81f8ad2698..a4a63aa9ebd9 100644 --- a/supervisely/serve/config.json +++ b/supervisely/serve/config.json @@ -11,7 +11,7 @@ "serve" ], "description": "Deploy model as REST API service", - "docker_image": "supervisely/yolov5:1.0.3", + "docker_image": "supervisely/yolov5:1.0.4", "instance_version": "6.8.88", "entrypoint": "python -m uvicorn main:m.app --app-dir ./supervisely/serve/src --host 0.0.0.0 --port 8000 --ws websockets", "port": 8000, diff --git a/supervisely/train/config.json b/supervisely/train/config.json index f7f1f56ab8e0..4130fa701959 100644 --- a/supervisely/train/config.json +++ b/supervisely/train/config.json @@ -10,7 +10,7 @@ "train" ], "description": "Dashboard to configure and monitor training", - "docker_image": "supervisely/yolov5:1.0.3", + "docker_image": "supervisely/yolov5:1.0.4", "min_instance_version": "6.8.70", "main_script": "supervisely/train/src/sly_train.py", "gui_template": "supervisely/train/src/gui.html",