From cb425b0c1bfbacc1db793c4f0bbd40ff90988beb Mon Sep 17 00:00:00 2001 From: Max Kolomeychenko Date: Fri, 7 May 2021 14:28:54 +0300 Subject: [PATCH] update UI + latest yolov5 sources (#15) * merge latest version done, not tested * split tabs with radio buttons * models table -wip * models table -wip * start split html template to parts * ui refactoring * compile-template wip - paths confusion * compile wip * train/val splits * keep/ignore unlabeled images * models table * training hyperparameters * UI templates - done * unlabeled count in UI * add adam optimizer * convert_project to detection - works * start train/val splits * splits wip * splits done, only simple tests * splits validation * data preprocessing - not tested * download weights - wip * init_script_arguments - not tested * init_script_arguments - not tested * prepare weights - wip * not tested * add metrics period * set output * artifacts dirs * train_batches_uploaded flag * pre-release for debug * update config --- requirements.txt | 12 +- supervisely/train/config.json | 2 +- supervisely/train/debug.env | 11 +- supervisely/train/src/gui.html | 355 ++---------------- supervisely/train/src/sly_init_ui.py | 153 -------- supervisely/train/src/sly_metrics.py | 45 +-- supervisely/train/src/sly_train.py | 124 +++--- supervisely/train/src/sly_train_globals.py | 39 +- supervisely/train/src/sly_train_utils.py | 55 +-- supervisely/train/src/sly_train_val_split.py | 55 --- supervisely/train/src/ui/architectures.html | 46 +++ supervisely/train/src/ui/architectures.py | 152 ++++++++ supervisely/train/src/ui/artifacts.html | 14 + supervisely/train/src/ui/artifacts.py | 17 + supervisely/train/src/ui/classes.html | 27 ++ supervisely/train/src/ui/classes.py | 25 ++ supervisely/train/src/ui/hyperparameters.html | 75 ++++ supervisely/train/src/ui/hyperparameters.py | 20 + supervisely/train/src/ui/input_project.html | 8 + supervisely/train/src/ui/input_project.py | 8 + supervisely/train/src/ui/monitoring.html | 115 ++++++ supervisely/train/src/ui/monitoring.py | 44 +++ supervisely/train/src/ui/splits.html | 133 +++++++ supervisely/train/src/ui/splits.py | 71 ++++ supervisely/train/src/ui/ui.py | 18 + .../{sly_prepare_data.py => yolov5_format.py} | 44 ++- train.py | 25 +- 27 files changed, 989 insertions(+), 704 deletions(-) delete mode 100644 supervisely/train/src/sly_init_ui.py delete mode 100644 supervisely/train/src/sly_train_val_split.py create mode 100644 supervisely/train/src/ui/architectures.html create mode 100644 supervisely/train/src/ui/architectures.py create mode 100644 supervisely/train/src/ui/artifacts.html create mode 100644 supervisely/train/src/ui/artifacts.py create mode 100644 supervisely/train/src/ui/classes.html create mode 100644 supervisely/train/src/ui/classes.py create mode 100644 supervisely/train/src/ui/hyperparameters.html create mode 100644 supervisely/train/src/ui/hyperparameters.py create mode 100644 supervisely/train/src/ui/input_project.html create mode 100644 supervisely/train/src/ui/input_project.py create mode 100644 supervisely/train/src/ui/monitoring.html create mode 100644 supervisely/train/src/ui/monitoring.py create mode 100644 supervisely/train/src/ui/splits.html create mode 100644 supervisely/train/src/ui/splits.py create mode 100644 supervisely/train/src/ui/ui.py rename supervisely/train/src/{sly_prepare_data.py => yolov5_format.py} (57%) diff --git a/requirements.txt b/requirements.txt index 92519641ad8a..3bd62b213f07 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # pip install -r requirements.txt -supervisely==6.1.52 +supervisely==6.1.74 # base ---------------------------------------- #matplotlib>=3.2.2 @@ -8,21 +8,17 @@ supervisely==6.1.52 #Pillow PyYAML>=5.3.1 #scipy>=1.4.1 -torch>=1.7.0 -torchvision>=0.8.1 +#torch>=1.7.0 +#torchvision>=0.8.1 tqdm>=4.41.0 # logging ------------------------------------- tensorboard>=2.4.1 # wandb -# logging ------------------------------------- -tensorboard>=2.4.1 -# wandb - # plotting ------------------------------------ seaborn==0.11.1 -pandas +#pandas # export -------------------------------------- # coremltools>=4.1 diff --git a/supervisely/train/config.json b/supervisely/train/config.json index 3dcf3042b0ab..f92022b6b8b9 100644 --- a/supervisely/train/config.json +++ b/supervisely/train/config.json @@ -19,5 +19,5 @@ "context_root": "Neural Networks", "context_category": "YOLO v5" }, - "instance_version": "6.3.2" + "instance_version": "6.4.14" } diff --git a/supervisely/train/debug.env b/supervisely/train/debug.env index 561feaa23f0f..cb6f81ed7bb9 100644 --- a/supervisely/train/debug.env +++ b/supervisely/train/debug.env @@ -2,14 +2,13 @@ PYTHONUNBUFFERED=1 DEBUG_APP_DIR="/app_debug_data" DEBUG_CACHE_DIR="/app_cache" +LOG_LEVEL="debug" -TASK_ID=2391 +TASK_ID=4326 -context.teamId=7 -context.workspaceId=263 -#modal.state.slyProjectId=1843 # coco-128 -modal.state.slyProjectId=1805 # lemons-annotated -#modal.state.slyFile="put your value here" +context.teamId=229 +context.workspaceId=287 +modal.state.slyProjectId=2128 # lemons-annotated SERVER_ADDRESS="put your value here" API_TOKEN="put your value here" diff --git a/supervisely/train/src/gui.html b/supervisely/train/src/gui.html index 463bfe2816ab..be2bb8140293 100644 --- a/supervisely/train/src/gui.html +++ b/supervisely/train/src/gui.html @@ -1,317 +1,40 @@ -
- - - {{data.projectName}} ({{data.projectImagesCount}} images) - - - - - - - - - - - - - - - - - - - Random - Based on image tags (not implemented yet) - Train = Val (not implemented yet) - - -
- - - - - - - - - - - -
-
- - - - - - - - -
If image does not have such tags, it will be assigned to training set
-
-
- All images are in both training and validation sets -
- - - - - - Pretrained on COCO - From custom model - - - -
- - - - - - - -
-
- - - -
- - - - - - - - - - - - - - - - - Multi-scale - - - Single class - - - - - - - - - -
-
Training hyperparameters templates:
- - scratch - finetune - - Restore - Defaults - -
-
Edit settings in YAML format:
- - - - - - - - Start training - -
- 0 training classes are selected -
-
- Path to model weights is not defined -
- -
-
{{data.progressName}}: {{data.currentProgressLabel}} / - {{data.totalProgressLabel}} -
- -
- - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - -
-
- - - - -
-
- - - - - - - - - - -
- Link to the directory with training artifacts will be here once training is finished -
- - - {{data.outputName}} - - - - - - +
+ + #yolov5-train .el-tabs.el-tabs-cards .el-radio { + display: flex; + align-items: start; + /*margin-bottom: 10px;*/ + margin-left: 0; + white-space: normal; + } + + #yolov5-train .el-tabs.el-tabs-cards .el-radio__label div { + color: #7f858e; + font-size: 13px; + } + + .beautiful-table { border-collapse: collapse; } + .beautiful-table tr:nth-child(2n) { background-color: #f6f8fa; } + .beautiful-table td, .beautiful-table th { + border: 1px solid #dfe2e5; + padding: 6px 13px; + text-align: center; + line-height: 20px; + } + + #yolov5-train .el-tabs.el-tabs-cards { border-radius: 4px; box-shadow: none; } + #yolov5-train .el-tabs.el-tabs-cards .el-tabs__header { background-color: #f6fafd; } + #yolov5-train .el-tabs.el-tabs-cards .el-tabs__nav { float: none; display: flex; justify-content: + space-between; } + #yolov5-train .el-tabs.el-tabs-cards .el-tabs__item { flex: 1; margin-bottom: -3px; padding: 9px 16px 13px; + height: auto; line-height: normal; border-radius: 4px; } + + + {% include 'supervisely/train/src/ui/input_project.html' %} + {% include 'supervisely/train/src/ui/classes.html' %} + {% include 'supervisely/train/src/ui/splits.html' %} + {% include 'supervisely/train/src/ui/architectures.html' %} + {% include 'supervisely/train/src/ui/hyperparameters.html' %} + {% include 'supervisely/train/src/ui/monitoring.html' %} + {% include 'supervisely/train/src/ui/artifacts.html' %}
\ No newline at end of file diff --git a/supervisely/train/src/sly_init_ui.py b/supervisely/train/src/sly_init_ui.py deleted file mode 100644 index a36940f5a4c2..000000000000 --- a/supervisely/train/src/sly_init_ui.py +++ /dev/null @@ -1,153 +0,0 @@ -import os -import supervisely_lib as sly - -import sly_train_globals as globals -import sly_metrics as metrics - - -empty_gallery = { - "content": { - "projectMeta": sly.ProjectMeta().to_json(), - "annotations": {}, - "layout": [] - } -} - - -def init_input_project(data, project_info): - data["projectId"] = globals.project_id - data["projectName"] = project_info.name - data["projectImagesCount"] = project_info.items_count - data["projectPreviewUrl"] = globals.api.image.preview_url(project_info.reference_image_url, 100, 100) - - -def init_classes_stats(data, state, project_meta): - stats = globals.api.project.get_stats(globals.project_id) - class_images = {} - for item in stats["images"]["objectClasses"]: - class_images[item["objectClass"]["name"]] = item["total"] - class_objects = {} - for item in stats["objects"]["items"]: - class_objects[item["objectClass"]["name"]] = item["total"] - - classes_json = project_meta.obj_classes.to_json() - for obj_class in classes_json: - obj_class["imagesCount"] = class_images[obj_class["title"]] - obj_class["objectsCount"] = class_objects[obj_class["title"]] - - data["classes"] = classes_json - state["selectedClasses"] = [] - - state["classes"] = len(classes_json) * [True] - - -def init_random_split(PROJECT, data, state): - data["randomSplit"] = [ - {"name": "train", "type": "success"}, - {"name": "val", "type": "primary"}, - {"name": "total", "type": "gray"}, - ] - data["totalImagesCount"] = PROJECT.items_count - - train_percent = 80 - train_count = int(PROJECT.items_count / 100 * train_percent) - state["randomSplit"] = { - "count": { - "total": PROJECT.items_count, - "train": train_count, - "val": PROJECT.items_count - train_count - }, - "percent": { - "total": 100, - "train": train_percent, - "val": 100 - train_percent - }, - "shareImagesBetweenSplits": False, - "sliderDisabled": False, - } - - state["splitMethod"] = 1 - state["trainTagName"] = "" - state["valTagName"] = "" - - -def init_model_settings(data, state): - data["modelSizes"] = [ - {"label": "yolov5s", "config": "yolov5s.yaml", "params": "7.3M"}, - {"label": "yolov5m", "config": "yolov5m.yaml", "params": "21.4M"}, - {"label": "yolov5l", "config": "yolov5l.yaml", "params": "47.0M"}, - {"label": "yolov5x", "config": "yolov5x.yaml", "params": "87.7M"}, - ] - state["modelSize"] = data["modelSizes"][0]["label"] - state["modelWeightsOptions"] = 1 - state["pretrainedWeights"] = f'{data["modelSizes"][0]["label"]}.pt' - - # @TODO: for debug - #state["weightsPath"] = "/yolov5_train/coco128_002/2390/weights/best.pt" - state["weightsPath"] = "" - - -def init_training_hyperparameters(state): - state["epochs"] = 10 - state["batchSize"] = 16 - state["imgSize"] = 640 - state["multiScale"] = False - state["singleClass"] = False - state["device"] = '0' - state["workers"] = 8 # 0 - for debug - state["activeTabName"] = "General" - state["hyp"] = { - "scratch": globals.scratch_str, - "finetune": globals.finetune_str, - } - state["hypRadio"] = "scratch" - - -def init_start_state(state): - state["started"] = False - state["activeNames"] = [] - - -def init_galleries(data): - data["vis"] = empty_gallery - data["labelsVis"] = empty_gallery - data["predVis"] = empty_gallery - data["syncBindings"] = [] - - -def init_progress(data): - data["progressName"] = "" - data["currentProgress"] = 0 - data["totalProgress"] = 0 - data["currentProgressLabel"] = "" - data["totalProgressLabel"] = "" - - -def init_output(data): - data["outputUrl"] = "" - data["outputName"] = "" - - -def init(data, state): - init_input_project(data, globals.project_info) - init_classes_stats(data, state, globals.project_meta) - init_random_split(globals.project_info, data, state) - init_model_settings(data, state) - init_training_hyperparameters(state) - init_start_state(state) - init_galleries(data) - init_progress(data) - init_output(data) - metrics.init(data, state) - - -def set_output(): - file_info = globals.api.file.get_info_by_path(globals.team_id, - os.path.join(globals.remote_artifacts_dir, 'results.png')) - fields = [ - {"field": "data.outputUrl", "payload": globals.api.file.get_url(file_info.id)}, - {"field": "data.outputName", "payload": globals.remote_artifacts_dir}, - ] - globals.api.app.set_fields(globals.task_id, fields) - globals.api.task.set_output_directory(globals.task_id, file_info.id, globals.remote_artifacts_dir) - diff --git a/supervisely/train/src/sly_metrics.py b/supervisely/train/src/sly_metrics.py index dd6f3b7ab800..594e96774826 100644 --- a/supervisely/train/src/sly_metrics.py +++ b/supervisely/train/src/sly_metrics.py @@ -1,5 +1,5 @@ import supervisely_lib as sly -import sly_train_globals as globals +import supervisely.train.src.sly_train_globals as globals def init_chart(title, names, xs, ys, smoothing=None): @@ -24,19 +24,19 @@ def init_chart(title, names, xs, ys, smoothing=None): def init(data, state): demo_x = [[], []] #[[1, 2, 3, 4], [2, 4, 6, 8]] demo_y = [[], []] #[[10, 15, 13, 17], [16, 5, 11, 9]] - data["mBox"] = init_chart("Box Loss", - names=["train", "val"], - xs=demo_x, - ys=demo_y, - smoothing=0.6) + data["mGIoU"] = init_chart("GIoU", + names=["train", "val"], + xs=demo_x, + ys=demo_y, + smoothing=0.6) - data["mObjectness"] = init_chart("Obj Loss", + data["mObjectness"] = init_chart("Objectness", names=["train", "val"], xs=demo_x, ys=demo_y, smoothing=0.6) - data["mClassification"] = init_chart("Cls Loss", + data["mClassification"] = init_chart("Classification", names=["train", "val"], xs=demo_x, ys=demo_y, @@ -54,23 +54,24 @@ def init(data, state): state["smoothing"] = 0.6 -def send_metrics(epoch, epochs, metrics): +def send_metrics(epoch, epochs, metrics, log_period=1): sly.logger.debug(f"Metrics: epoch {epoch} / {epochs}", extra={"metrics": metrics}) - fields = [ - {"field": "data.mBox.series[0].data", "payload": [[epoch, metrics["train/box_loss"]]], "append": True}, - {"field": "data.mBox.series[1].data", "payload": [[epoch, metrics["val/box_loss"]]], "append": True}, + if epoch % log_period == 0 or epoch == epochs: + fields = [ + {"field": "data.mGIoU.series[0].data", "payload": [[epoch, metrics["train/box_loss"]]], "append": True}, + {"field": "data.mGIoU.series[1].data", "payload": [[epoch, metrics["val/box_loss"]]], "append": True}, - {"field": "data.mObjectness.series[0].data", "payload": [[epoch, metrics["train/obj_loss"]]], "append": True}, - {"field": "data.mObjectness.series[1].data", "payload": [[epoch, metrics["val/obj_loss"]]], "append": True}, + {"field": "data.mObjectness.series[0].data", "payload": [[epoch, metrics["train/obj_loss"]]], "append": True}, + {"field": "data.mObjectness.series[1].data", "payload": [[epoch, metrics["val/obj_loss"]]], "append": True}, - {"field": "data.mClassification.series[0].data", "payload": [[epoch, metrics["train/cls_loss"]]], "append": True}, - {"field": "data.mClassification.series[1].data", "payload": [[epoch, metrics["val/cls_loss"]]], "append": True}, + {"field": "data.mClassification.series[0].data", "payload": [[epoch, metrics["train/cls_loss"]]], "append": True}, + {"field": "data.mClassification.series[1].data", "payload": [[epoch, metrics["val/cls_loss"]]], "append": True}, - {"field": "data.mPR.series[0].data", "payload": [[epoch, metrics["metrics/precision"]]], "append": True}, - {"field": "data.mPR.series[1].data", "payload": [[epoch, metrics["metrics/recall"]]], "append": True}, + {"field": "data.mPR.series[0].data", "payload": [[epoch, metrics["metrics/precision"]]], "append": True}, + {"field": "data.mPR.series[1].data", "payload": [[epoch, metrics["metrics/recall"]]], "append": True}, - {"field": "data.mMAP.series[0].data", "payload": [[epoch, metrics["metrics/mAP_0.5"]]], "append": True}, - {"field": "data.mMAP.series[1].data", "payload": [[epoch, metrics["metrics/mAP_0.5:0.95"]]], "append": True}, - ] - globals.api.app.set_fields(globals.task_id, fields) + {"field": "data.mMAP.series[0].data", "payload": [[epoch, metrics["metrics/mAP_0.5"]]], "append": True}, + {"field": "data.mMAP.series[1].data", "payload": [[epoch, metrics["metrics/mAP_0.5:0.95"]]], "append": True}, + ] + globals.api.app.set_fields(globals.task_id, fields) diff --git a/supervisely/train/src/sly_train.py b/supervisely/train/src/sly_train.py index 77d77c00c691..82246c1eaf99 100644 --- a/supervisely/train/src/sly_train.py +++ b/supervisely/train/src/sly_train.py @@ -1,23 +1,20 @@ import os import supervisely_lib as sly -from sly_train_globals import init_project_info_and_meta, \ - my_app, task_id, \ - team_id, workspace_id, project_id, \ - root_source_path, scratch_str, finetune_str - -# to import correct values -# project_info, project_meta, \ -# local_artifacts_dir, remote_artifacts_dir -import sly_train_globals as g - -from sly_train_val_split import train_val_split -import sly_init_ui as ui -from sly_prepare_data import filter_and_transform_labels -from sly_train_utils import init_script_arguments -from sly_utils import get_progress_cb, upload_artifacts +import supervisely.train.src.sly_train_globals as g +from supervisely.train.src.sly_train_globals import \ + my_app, task_id, \ + team_id, workspace_id, project_id, \ + root_source_dir, scratch_str, finetune_str +import ui.ui as ui +from sly_train_utils import init_script_arguments +from sly_utils import get_progress_cb, upload_artifacts +from supervisely.train.src.ui.splits import get_train_val_sets, verify_train_val_sets +import supervisely.train.src.yolov5_format as yolov5_format +from supervisely.train.src.ui.architectures import prepare_weights +from supervisely.train.src.ui.artifacts import set_task_output import train as train_yolov5 @@ -33,48 +30,51 @@ def restore_hyp(api: sly.Api, task_id, context, state, app_logger): @my_app.callback("train") @sly.timeit def train(api: sly.Api, task_id, context, state, app_logger): - api.app.set_field(task_id, "state.activeNames", ["labels", "train", "pred", "metrics"]) #"logs", - - # prepare directory for original Supervisely project - project_dir = os.path.join(my_app.data_dir, "sly_project") - sly.fs.mkdir(project_dir) - sly.fs.clean_dir(project_dir) # useful for debug, has no effect in production - - # download Sypervisely project (using cache) - sly.download_project(api, project_id, project_dir, cache=my_app.cache, - progress_cb=get_progress_cb("Download data (using cache)", g.project_info.items_count * 2)) - - # prepare directory for transformed data (nn will use it for training) - yolov5_format_dir = os.path.join(my_app.data_dir, "train_data") - sly.fs.mkdir(yolov5_format_dir) - sly.fs.clean_dir(yolov5_format_dir) # useful for debug, has no effect in production - - # split data to train/val sets, filter objects by classes, convert Supervisely project to YOLOv5 format(COCO) - train_split, val_split = train_val_split(project_dir, state) - train_classes = state["selectedClasses"] - progress_cb = get_progress_cb("Convert Supervisely to YOLOv5 format", g.project_info.items_count) - filter_and_transform_labels(project_dir, train_classes, train_split, val_split, yolov5_format_dir, progress_cb) - - # download initial weights from team files - if state["modelWeightsOptions"] == 2: # transfer learning from custom weights - weights_path_remote = state["weightsPath"] - weights_path_local = os.path.join(my_app.data_dir, sly.fs.get_file_name_with_ext(weights_path_remote)) - file_info = api.file.get_info_by_path(team_id, weights_path_remote) - api.file.download(team_id, weights_path_remote, weights_path_local, my_app.cache, - progress_cb=get_progress_cb("Download weights", file_info.sizeb, is_size=True)) - - # init sys.argv for main training script - init_script_arguments(state, yolov5_format_dir, g.project_info.name) - - # start train script - get_progress_cb("YOLOv5: Scanning data ", 1)(1) - train_yolov5.main() - - # upload artifacts directory to Team Files - upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir) - - # show path to the artifacts directory in Team Files - ui.set_output() + try: + prepare_weights(state) + + # prepare directory for original Supervisely project + project_dir = os.path.join(my_app.data_dir, "sly_project") + sly.fs.mkdir(project_dir, remove_content_if_exists=True) # clean content for debug, has no effect in prod + + # download and preprocess Sypervisely project (using cache) + download_progress = get_progress_cb("Download data (using cache)", g.project_info.items_count * 2) + sly.download_project(api, project_id, project_dir, cache=my_app.cache, progress_cb=download_progress) + + # preprocessing: transform labels to bboxes, filter classes, ... + sly.Project.to_detection_task(project_dir, inplace=True) + train_classes = state["selectedClasses"] + sly.Project.remove_classes_except(project_dir, classes_to_keep=train_classes, inplace=True) + if state["unlabeledImages"] == "ignore": + sly.Project.remove_items_without_objects(project_dir, inplace=True) + + # split to train / validation sets (paths to images and annotations) + train_set, val_set = get_train_val_sets(project_dir, state) + verify_train_val_sets(train_set, val_set) + + # prepare directory for data in YOLOv5 format (nn will use it for training) + train_data_dir = os.path.join(my_app.data_dir, "train_data") + sly.fs.mkdir(train_data_dir, remove_content_if_exists=True) # clean content for debug, has no effect in prod + + # convert Supervisely project to YOLOv5 format + progress_cb = get_progress_cb("Convert Supervisely to YOLOv5 format", len(train_set) + len(val_set)) + yolov5_format.transform(project_dir, train_data_dir, train_set, val_set, progress_cb) + + # init sys.argv for main training script + init_script_arguments(state, train_data_dir, g.project_info.name) + + # start train script + api.app.set_field(task_id, "state.activeNames", ["labels", "train", "pred", "metrics"]) # "logs", + get_progress_cb("YOLOv5: Scanning data ", 1)(1) + train_yolov5.main() + + # upload artifacts directory to Team Files + upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir) + set_task_output() + except Exception as e: + my_app.show_modal_window(f"Oops! Something went wrong, please try again or contact tech support. " + f"Find more info in the app logs. Error: {repr(e)}", level="error") + api.app.set_field(task_id, "state.started", False) # stop application get_progress_cb("Finished, app is stopped automatically", 1)(1) @@ -92,8 +92,7 @@ def main(): state = {} data["taskId"] = task_id - # read project information and meta (classes + tags) - init_project_info_and_meta() + my_app.compile_template(g.root_source_dir) # init data for UI widgets ui.init(data, state) @@ -101,13 +100,8 @@ def main(): my_app.run(data=data, state=state) -# @TODO: change pip requirements to quickly skip them (already installed) -# @TODO: handle soft stop event - # New features: -# @TODO: adam or SGD opt? -# @TODO: train == val - handle case in data_config.yaml to avoid data duplication # @TODO: resume training -# @TODO: repeat dataset (for small lemons) +# @TODO: save checkpoint every N-th epochs if __name__ == "__main__": sly.main_wrapper("main", main) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 549a8976c3f9..00230a36a2e7 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -14,30 +14,31 @@ local_artifacts_dir = None remote_artifacts_dir = None +project_info = api.project.get_info_by_id(project_id) +project_meta = sly.ProjectMeta.from_json(api.project.get_meta(project_id)) -project_info = None -project_meta = None +root_source_dir = str(Path(sys.argv[0]).parents[3]) +sly.logger.info(f"Root source directory: {root_source_dir}") +sys.path.append(root_source_dir) -root_source_path = str(Path(sys.argv[0]).parents[3]) -sly.logger.info(f"Root source directory: {root_source_path}") -sys.path.append(root_source_path) +source_path = str(Path(sys.argv[0]).parents[0]) +sly.logger.info(f"Source directory: {source_path}") +sys.path.append(source_path) -# script_path = str(Path(sys.argv[0]).parents[3])) -# root_app_dir = script_path.parent.parent.absolute() -# sly.logger.info(f"Root app directory: {root_app_dir}") -# sys.path.append(root_app_dir) +with open(os.path.join(root_source_dir, "data/hyp.scratch.yaml"), 'r') as file: + scratch_str = file.read() # yaml.safe_load( -def init_project_info_and_meta(): - global project_info, project_meta - project_info = api.project.get_info_by_id(project_id) - project_meta_json = api.project.get_meta(project_id) - project_meta = sly.ProjectMeta.from_json(project_meta_json) - +with open(os.path.join(root_source_dir, "data/hyp.finetune.yaml"), 'r') as file: + finetune_str = file.read() # yaml.safe_load( -with open(os.path.join(root_source_path, "data/hyp.scratch.yaml"), 'r') as file: - scratch_str = file.read() # yaml.safe_load( -with open(os.path.join(root_source_path, "data/hyp.finetune.yaml"), 'r') as file: - finetune_str = file.read() # yaml.safe_load( \ No newline at end of file +runs_dir = os.path.join(my_app.data_dir, 'runs') +sly.fs.mkdir(runs_dir, remove_content_if_exists=True) # for debug, does nothing in production +experiment_name = str(task_id) +local_artifacts_dir = os.path.join(runs_dir, experiment_name) +sly.logger.info(f"All training artifacts will be saved to local directory {local_artifacts_dir}") +remote_artifacts_dir = os.path.join("/yolov5_train", project_info.name, experiment_name) +remote_artifacts_dir = api.file.get_free_dir_name(team_id, remote_artifacts_dir) +sly.logger.info(f"After training artifacts will be uploaded to Team Files: {remote_artifacts_dir}") \ No newline at end of file diff --git a/supervisely/train/src/sly_train_utils.py b/supervisely/train/src/sly_train_utils.py index 0d7f6a0e3570..80e7db28d4a3 100644 --- a/supervisely/train/src/sly_train_utils.py +++ b/supervisely/train/src/sly_train_utils.py @@ -11,28 +11,23 @@ def init_script_arguments(state, yolov5_format_dir, input_project_name): global local_artifacts_dir, remote_artifacts_dir + sys.argv.append("--sly") + data_path = os.path.join(yolov5_format_dir, 'data_config.yaml') sys.argv.extend(["--data", data_path]) - try: - hyp_content = yaml.safe_load(state["hyp"][state["hypRadio"]]) - hyp = os.path.join(my_app.data_dir, 'hyp.custom.yaml') - with open(hyp, 'w') as f: - f.write(state["hyp"][state["hypRadio"]]) - except yaml.YAMLError as e: - sly.logger.error(repr(e)) - api.app.set_field(task_id, "state.started", False) - return + hyp_content = yaml.safe_load(state["hyp"][state["hypRadio"]]) + hyp = os.path.join(my_app.data_dir, 'hyp.custom.yaml') + with open(hyp, 'w') as f: + f.write(state["hyp"][state["hypRadio"]]) sys.argv.extend(["--hyp", hyp]) - weights = "" # random (not tested) - if state["modelWeightsOptions"] == 1: - weights = state["pretrainedWeights"] - cfg = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../models', f"{state['modelSize']}.yaml") + if state["weightsInitialization"] == "coco": + model_name = state['selectedModel'].lower() + _sub_path = "models/hub" if model_name.endswith('6') else "models" + cfg = os.path.join(g.root_source_dir, _sub_path, f"{model_name}.yaml") sys.argv.extend(["--cfg", cfg]) - elif state["modelWeightsOptions"] == 2: - weights = state["weightsPath"] - sys.argv.extend(["--weights", weights]) + sys.argv.extend(["--weights", state["weightsPath"]]) sys.argv.extend(["--epochs", str(state["epochs"])]) sys.argv.extend(["--batch-size", str(state["batchSize"])]) @@ -45,30 +40,12 @@ def init_script_arguments(state, yolov5_format_dir, input_project_name): if "workers" in state: sys.argv.extend(["--workers", str(state["workers"])]) + if state["optimizer"] == "Adam": + sys.argv.append("--adam") - training_dir = os.path.join(my_app.data_dir, 'experiment', input_project_name) - experiment_name = str(task_id) - local_artifacts_dir = os.path.join(training_dir, experiment_name) - _exp_index = 1 - while sly.fs.dir_exists(local_artifacts_dir): - experiment_name = "{}_{:03d}".format(task_id, _exp_index) - local_artifacts_dir = os.path.join(training_dir, experiment_name) - _exp_index += 1 - g.local_artifacts_dir = local_artifacts_dir - - sys.argv.extend(["--project", training_dir]) - sys.argv.extend(["--name", experiment_name]) - - sys.argv.append("--sly") - - remote_experiment_name = str(task_id) - remote_artifacts_dir = os.path.join("/yolov5_train", input_project_name, remote_experiment_name) - _exp_index = 1 - while api.file.dir_exists(team_id, remote_artifacts_dir): - remote_experiment_name = "{}_{:03d}".format(task_id, _exp_index) - remote_artifacts_dir = os.path.join("/yolov5_train", input_project_name, remote_experiment_name) - _exp_index += 1 - g.remote_artifacts_dir = remote_artifacts_dir + sys.argv.extend(["--metrics_period", str(state["metricsPeriod"])]) + sys.argv.extend(["--project", g.runs_dir]) + sys.argv.extend(["--name", g.experiment_name]) def send_epoch_log(epoch, epochs, progress): diff --git a/supervisely/train/src/sly_train_val_split.py b/supervisely/train/src/sly_train_val_split.py deleted file mode 100644 index e9807c5c3b0a..000000000000 --- a/supervisely/train/src/sly_train_val_split.py +++ /dev/null @@ -1,55 +0,0 @@ -import random -import supervisely_lib as sly - - -def _list_items(project_dir): - items = [] - project = sly.Project(project_dir, sly.OpenMode.READ) - for dataset in project: - for item_name in dataset: - items.append((dataset.name, item_name)) - return items - - -def _split_random(project_dir, train_count, val_count): - items = _list_items(project_dir) - random.shuffle(items) - train_items = items[:train_count] - val_items = items[train_count:] - if len(val_items) != val_count: - sly.logger.warn("Issue in train/val random split in GUI", extra={ - "train_count": train_count, - "val_count": val_count, - "items_count": len(items), - "train_count + val_count": train_count + val_count - }) - #raise RuntimeError("Incorrect train/val random split") - return train_items, val_items - - -def _split_same(project_dir): - items = _list_items(project_dir) - return items, items.copy() - - -def _split_tags(project_dir, train_tag_name, val_tag_name): - raise NotImplementedError() - - -def train_val_split(project_dir, state): - split_method = state["splitMethod"] - train_count = state["randomSplit"]["count"]["train"] - val_count = state["randomSplit"]["count"]["val"] - - train_split = None - val_split = None - if split_method == 1: # Random - train_split, val_split = _split_random(project_dir, train_count, val_count) - elif split_method == 2: # Based on image tags - train_split, val_split = _split_tags() - elif split_method == 3: # Train = Val - train_split, val_split = _split_same() - else: - raise ValueError(f"Train/val split method: {split_method} unknown") - - return train_split, val_split \ No newline at end of file diff --git a/supervisely/train/src/ui/architectures.html b/supervisely/train/src/ui/architectures.html new file mode 100644 index 000000000000..4b31235c18d9 --- /dev/null +++ b/supervisely/train/src/ui/architectures.html @@ -0,0 +1,46 @@ + + + + + Pretrained on COCO +
Default pretrained checkpoints provided by authors of YOLOv5
+ + + + + + + + + + + + +
+
+
+ {{row["subtitle"]}} +
+
+
+ + {{model[column.key]}} + +
+
+ {{model[column.key]}} +
+
+ + + + Custom weights + + + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/architectures.py b/supervisely/train/src/ui/architectures.py new file mode 100644 index 000000000000..ab00109ee5dd --- /dev/null +++ b/supervisely/train/src/ui/architectures.py @@ -0,0 +1,152 @@ +import errno +import os +import supervisely.train.src.sly_train_globals as g +from supervisely.train.src.sly_utils import get_progress_cb +import supervisely_lib as sly + + +def get_models_list(): + return [ + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5s", + "Size": 640, + "mAP^val": 36.7, + "mAP^test": 36.7, + "mAP^val_0.5": 55.4, + "Speed": 2.0, + "Params": 7.3, + "FLOPS": 17.0, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5m", + "Size": 640, + "mAP^val": 44.5, + "mAP^test": 44.5, + "mAP^val_0.5": 63.1, + "Speed": 2.7, + "Params": 21.4, + "FLOPS": 51.3, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5l", + "Size": 640, + "mAP^val": 48.2, + "mAP^test": 48.2, + "mAP^val_0.5": 66.9, + "Speed": 3.8, + "Params": 47.0, + "FLOPS": 115.4, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5x", + "Size": 640, + "mAP^val": 50.4, + "mAP^test": 50.4, + "mAP^val_0.5": 68.8, + "Speed": 6.1, + "Params": 87.7, + "FLOPS": 218.8, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5s6", + "Size": 1280, + "mAP^val": 43.3, + "mAP^test": 43.3, + "mAP^val_0.5": 61.9, + "Speed": 4.3, + "Params": 12.7, + "FLOPS": 17.4, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5m6", + "Size": 1280, + "mAP^val": 50.5, + "mAP^test": 50.5, + "mAP^val_0.5": 68.7, + "Speed": 8.4, + "Params": 35.9, + "FLOPS": 52.4, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5l6", + "Size": 1280, + "mAP^val": 53.4, + "mAP^test": 53.4, + "mAP^val_0.5": 71.1, + "Speed": 12.3, + "Params": 77.2, + "FLOPS": 117.7, + }, + { + "config": "", + "weightsUrl": "", + "Model": "YOLOv5x6", + "Size": 1280, + "mAP^val": 54.4, + "mAP^test": 54.4, + "mAP^val_0.5": 72.0, + "Speed": 22.4, + "Params": 141.8, + "FLOPS": 222.9, + }, + ] + + +def get_table_columns(): + return [ + {"key": "Model", "title": "Model", "subtitle": None}, + {"key": "Size", "title": "Size", "subtitle": "(pixels)"}, + {"key": "mAP^val", "title": "mAPval", "subtitle": "0.5:0.95"}, + {"key": "mAP^test", "title": "mAPtest", "subtitle": "0.5:0.95"}, + {"key": "mAP^val_0.5", "title": "mAPval", "subtitle": "0.5"}, + {"key": "Speed", "title": "Speed", "subtitle": "V100 (ms)"}, + {"key": "Params", "title": "Params", "subtitle": "(M)"}, + {"key": "FLOPS", "title": "FLOPS", "subtitle": "640 (B)"}, + ] + + +def init(data, state): + data["models"] = get_models_list() + data["modelColumns"] = get_table_columns() + state["selectedModel"] = "YOLOv5s" + state["weightsInitialization"] = "coco" + + # @TODO: for debug + #state["weightsPath"] = "/yolov5_train/coco128_002/2390/weights/best.pt" + state["weightsPath"] = "" + + +def prepare_weights(state): + if state["weightsInitialization"] == "custom": + # download custom weights + weights_path_remote = state["weightsPath"] + if not weights_path_remote.endswith(".pt"): + raise ValueError(f"Weights file has unsupported extension {sly.fs.get_file_ext(weights_path_remote)}. " + f"Supported: '.pt'") + weights_path_local = os.path.join(g.my_app.data_dir, sly.fs.get_file_name_with_ext(weights_path_remote)) + file_info = g.api.file.get_info_by_path(g.team_id, weights_path_remote) + if file_info is None: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), weights_path_remote) + progress_cb = get_progress_cb("Download weights", file_info.sizeb, is_size=True) + g.api.file.download(g.team_id, weights_path_remote, weights_path_local, g.my_app.cache, progress_cb) + + state["_weightsPath"] = weights_path_remote + state["weightsPath"] = weights_path_local + else: + model_name = state['selectedModel'].lower() + state["weightsPath"] = f"{model_name}.pt" + sly.logger.info("Pretrained COCO weights will be added automatically") diff --git a/supervisely/train/src/ui/artifacts.html b/supervisely/train/src/ui/artifacts.html new file mode 100644 index 000000000000..437aa169e248 --- /dev/null +++ b/supervisely/train/src/ui/artifacts.html @@ -0,0 +1,14 @@ + +
+ Link to the directory with training artifacts will be here once training is finished +
+ + + {{data.outputName}} + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/artifacts.py b/supervisely/train/src/ui/artifacts.py new file mode 100644 index 000000000000..16b946b32e5c --- /dev/null +++ b/supervisely/train/src/ui/artifacts.py @@ -0,0 +1,17 @@ +import os +import supervisely.train.src.sly_train_globals as g + + +def init(data): + data["outputUrl"] = None + data["outputName"] = None + + +def set_task_output(): + file_info = g.api.file.get_info_by_path(g.team_id, os.path.join(g.remote_artifacts_dir, 'results.png')) + fields = [ + {"field": "data.outputUrl", "payload": g.api.file.get_url(file_info.id)}, + {"field": "data.outputName", "payload": g.remote_artifacts_dir}, + ] + g.api.app.set_fields(g.task_id, fields) + g.api.task.set_output_directory(g.task_id, file_info.id, g.remote_artifacts_dir) \ No newline at end of file diff --git a/supervisely/train/src/ui/classes.html b/supervisely/train/src/ui/classes.html new file mode 100644 index 000000000000..145d4df23632 --- /dev/null +++ b/supervisely/train/src/ui/classes.html @@ -0,0 +1,27 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/classes.py b/supervisely/train/src/ui/classes.py new file mode 100644 index 000000000000..429e68d945bf --- /dev/null +++ b/supervisely/train/src/ui/classes.py @@ -0,0 +1,25 @@ +import supervisely_lib as sly + + +def init(api: sly.Api, data, state, project_id, project_meta: sly.ProjectMeta): + stats = api.project.get_stats(project_id) + class_images = {} + for item in stats["images"]["objectClasses"]: + class_images[item["objectClass"]["name"]] = item["total"] + class_objects = {} + for item in stats["objects"]["items"]: + class_objects[item["objectClass"]["name"]] = item["total"] + + classes_json = project_meta.obj_classes.to_json() + for obj_class in classes_json: + obj_class["imagesCount"] = class_images[obj_class["title"]] + obj_class["objectsCount"] = class_objects[obj_class["title"]] + + unlabeled_count = 0 + for ds_counter in stats["images"]["datasets"]: + unlabeled_count += ds_counter["imagesNotMarked"] + + data["classes"] = classes_json + state["selectedClasses"] = [] + state["classes"] = len(classes_json) * [True] + data["unlabeledCount"] = unlabeled_count \ No newline at end of file diff --git a/supervisely/train/src/ui/hyperparameters.html b/supervisely/train/src/ui/hyperparameters.html new file mode 100644 index 000000000000..f346fb020f35 --- /dev/null +++ b/supervisely/train/src/ui/hyperparameters.html @@ -0,0 +1,75 @@ + + + + + + + + + + + + Multi-scale + + + Single class + + + + + + + + +
+ Log metrics every + + epochs +
+ + + + + + + + + + + + Scratch mode +
Recommended hyperparameters for training from scratch
+ + + Restore Defaults + + + + + + Finetune mode +
Recommended hyperparameters for model finutuning
+ + + Restore Defaults + + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/hyperparameters.py b/supervisely/train/src/ui/hyperparameters.py new file mode 100644 index 000000000000..8042fba6d013 --- /dev/null +++ b/supervisely/train/src/ui/hyperparameters.py @@ -0,0 +1,20 @@ +import supervisely_lib as sly +import supervisely.train.src.sly_train_globals as g + + +def init(state): + state["epochs"] = 10 + state["batchSize"] = 16 + state["imgSize"] = 640 + state["multiScale"] = False + state["singleClass"] = False + state["device"] = '0' + state["workers"] = 8 # 0 - for debug @TODO: for debug + state["activeTabName"] = "General" + state["hyp"] = { + "scratch": g.scratch_str, + "finetune": g.finetune_str, + } + state["hypRadio"] = "scratch" + state["optimizer"] = "SGD" + state["metricsPeriod"] = 1 diff --git a/supervisely/train/src/ui/input_project.html b/supervisely/train/src/ui/input_project.html new file mode 100644 index 000000000000..ff0ef021bc63 --- /dev/null +++ b/supervisely/train/src/ui/input_project.html @@ -0,0 +1,8 @@ + + + {{data.projectName}} ({{data.projectImagesCount}} + images) + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/input_project.py b/supervisely/train/src/ui/input_project.py new file mode 100644 index 000000000000..e81d822f05c1 --- /dev/null +++ b/supervisely/train/src/ui/input_project.py @@ -0,0 +1,8 @@ +import supervisely.train.src.sly_train_globals as g + + +def init(data): + data["projectId"] = g.project_info.id + data["projectName"] = g.project_info.name + data["projectImagesCount"] = g.project_info.items_count + data["projectPreviewUrl"] = g.api.image.preview_url(g.project_info.reference_image_url, 100, 100) diff --git a/supervisely/train/src/ui/monitoring.html b/supervisely/train/src/ui/monitoring.html new file mode 100644 index 000000000000..26458aa203a6 --- /dev/null +++ b/supervisely/train/src/ui/monitoring.html @@ -0,0 +1,115 @@ + + + Start training + +
+ 0 training classes are selected +
+
+ Path to model weights is not defined +
+
+
{{data.progressName}}: {{data.currentProgressLabel}} / + {{data.totalProgressLabel}} +
+ +
+ + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+
+ + + + +
+
+ + + + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/monitoring.py b/supervisely/train/src/ui/monitoring.py new file mode 100644 index 000000000000..0dbe0048bfe5 --- /dev/null +++ b/supervisely/train/src/ui/monitoring.py @@ -0,0 +1,44 @@ +import supervisely_lib as sly +import supervisely.train.src.sly_metrics as metrics + + +empty_gallery = { + "content": { + "projectMeta": sly.ProjectMeta().to_json(), + "annotations": {}, + "layout": [] + } +} + + +def init(data, state): + _init_start_state(state) + _init_galleries(data) + _init_progress(data) + _init_output(data) + metrics.init(data, state) + + +def _init_start_state(state): + state["started"] = False + state["activeNames"] = [] + + +def _init_galleries(data): + data["vis"] = empty_gallery + data["labelsVis"] = empty_gallery + data["predVis"] = empty_gallery + data["syncBindings"] = [] + + +def _init_progress(data): + data["progressName"] = "" + data["currentProgress"] = 0 + data["totalProgress"] = 0 + data["currentProgressLabel"] = "" + data["totalProgressLabel"] = "" + + +def _init_output(data): + data["outputUrl"] = "" + data["outputName"] = "" \ No newline at end of file diff --git a/supervisely/train/src/ui/splits.html b/supervisely/train/src/ui/splits.html new file mode 100644 index 000000000000..a40deb164f45 --- /dev/null +++ b/supervisely/train/src/ui/splits.html @@ -0,0 +1,133 @@ + + + + + Random +
Shuffle data and split with defined probability
+ + + + + + + + + + + + + + + + Based on image tags +
Images should have assigned train or val tag
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Based on datasets +
Select one or several datasets for every split
+ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/supervisely/train/src/ui/splits.py b/supervisely/train/src/ui/splits.py new file mode 100644 index 000000000000..e7e8095988db --- /dev/null +++ b/supervisely/train/src/ui/splits.py @@ -0,0 +1,71 @@ +import supervisely_lib as sly + + +def init(project_info, project_meta: sly.ProjectMeta, data, state): + data["randomSplit"] = [ + {"name": "train", "type": "success"}, + {"name": "val", "type": "primary"}, + {"name": "total", "type": "gray"}, + ] + data["totalImagesCount"] = project_info.items_count + + train_percent = 80 + train_count = int(project_info.items_count / 100 * train_percent) + state["randomSplit"] = { + "count": { + "total": project_info.items_count, + "train": train_count, + "val": project_info.items_count - train_count + }, + "percent": { + "total": 100, + "train": train_percent, + "val": 100 - train_percent + }, + "shareImagesBetweenSplits": False, + "sliderDisabled": False, + } + + state["splitMethod"] = "random" + + state["trainTagName"] = "" + if project_meta.tag_metas.get("train") is not None: + state["trainTagName"] = "train" + state["valTagName"] = "" + if project_meta.tag_metas.get("val") is not None: + state["valTagName"] = "val" + + state["trainDatasets"] = [] + state["valDatasets"] = [] + + state["unlabeledImages"] = "keep" + state["untaggedImages"] = "train" + + +def get_train_val_sets(project_dir, state): + split_method = state["splitMethod"] + if split_method == "random": + train_count = state["randomSplit"]["count"]["train"] + val_count = state["randomSplit"]["count"]["val"] + train_set, val_set = sly.Project.get_train_val_splits_by_count(project_dir, train_count, val_count) + return train_set, val_set + elif split_method == "tags": + train_tag_name = state["trainTagName"] + val_tag_name = state["valTagName"] + add_untagged_to = state["untaggedImages"] + train_set, val_set = sly.Project.get_train_val_splits_by_tag(project_dir, train_tag_name, val_tag_name, add_untagged_to) + return train_set, val_set + elif split_method == "datasets": + train_datasets = state["trainDatasets"] + val_datasets = state["valDatasets"] + train_set, val_set = sly.Project.get_train_val_splits_by_dataset(project_dir, train_datasets, val_datasets) + return train_set, val_set + else: + raise ValueError(f"Unknown split method: {split_method}") + + +def verify_train_val_sets(train_set, val_set): + if len(train_set) == 0: + raise ValueError("Train set is empty, check or change split configuration") + if len(val_set) == 0: + raise ValueError("Val set is empty, check or change split configuration") \ No newline at end of file diff --git a/supervisely/train/src/ui/ui.py b/supervisely/train/src/ui/ui.py new file mode 100644 index 000000000000..8566f30013c3 --- /dev/null +++ b/supervisely/train/src/ui/ui.py @@ -0,0 +1,18 @@ +import supervisely.train.src.sly_train_globals as g +import supervisely.train.src.ui.input_project as input_project +import supervisely.train.src.ui.classes as training_classes +import supervisely.train.src.ui.splits as train_val_split +import supervisely.train.src.ui.architectures as model_architectures +import supervisely.train.src.ui.hyperparameters as hyperparameters +import supervisely.train.src.ui.monitoring as monitoring +import supervisely.train.src.ui.artifacts as artifacts + + +def init(data, state): + input_project.init(data) + training_classes.init(g.api, data, state, g.project_id, g.project_meta) + train_val_split.init(g.project_info, g.project_meta, data, state) + model_architectures.init(data, state) + hyperparameters.init(state) + monitoring.init(data, state) + artifacts.init(data) diff --git a/supervisely/train/src/sly_prepare_data.py b/supervisely/train/src/yolov5_format.py similarity index 57% rename from supervisely/train/src/sly_prepare_data.py rename to supervisely/train/src/yolov5_format.py index aab0e6671dcb..5e605e7255cd 100644 --- a/supervisely/train/src/sly_prepare_data.py +++ b/supervisely/train/src/yolov5_format.py @@ -3,7 +3,7 @@ import supervisely_lib as sly -def transform_label(class_names, img_size, label: sly.Label): +def _transform_label(class_names, img_size, label: sly.Label): class_number = class_names.index(label.obj_class.name) rect_geometry = label.geometry.to_bbox() center = rect_geometry.center @@ -15,13 +15,12 @@ def transform_label(class_names, img_size, label: sly.Label): return result -def _create_data_config(output_dir, meta: sly.ProjectMeta, keep_classes): +def _create_data_config(output_dir, meta: sly.ProjectMeta): class_names = [] class_colors = [] for obj_class in meta.obj_classes: - if obj_class.name in keep_classes: - class_names.append(obj_class.name) - class_colors.append(obj_class.color) + class_names.append(obj_class.name) + class_colors.append(obj_class.color) data_yaml = { "train": os.path.join(output_dir, "images/train"), @@ -44,11 +43,11 @@ def _create_data_config(output_dir, meta: sly.ProjectMeta, keep_classes): return data_yaml -def transform_annotation(ann, class_names, save_path): +def _transform_annotation(ann, class_names, save_path): yolov5_ann = [] for label in ann.labels: if label.obj_class.name in class_names: - yolov5_ann.append(transform_label(class_names, ann.img_size, label)) + yolov5_ann.append(_transform_label(class_names, ann.img_size, label)) with open(save_path, 'w') as file: file.write("\n".join(yolov5_ann)) @@ -67,9 +66,9 @@ def _process_split(project, class_names, images_dir, labels_dir, split, progress ann = sly.Annotation.from_json(ann_json, project.meta) save_ann_path = os.path.join(labels_dir, f"{sly.fs.get_file_name(item_name)}.txt") - empty = transform_annotation(ann, class_names, save_ann_path) + empty = _transform_annotation(ann, class_names, save_ann_path) if empty: - sly.logger.warning(f"Empty annotation dataset={dataset_name} image={item_name}") + sly.logger.warning(f"Empty annotation: dataset={dataset_name}, image={item_name}") img_path = dataset.get_img_path(item_name) save_img_path = os.path.join(images_dir, item_name) @@ -78,12 +77,25 @@ def _process_split(project, class_names, images_dir, labels_dir, split, progress progress_cb(len(batch)) -def filter_and_transform_labels(input_dir, train_classes, - train_split, val_split, - output_dir, progress_cb): - project = sly.Project(input_dir, sly.OpenMode.READ) - data_yaml = _create_data_config(output_dir, project.meta, train_classes) +def _transform_set(set_name, data_yaml, project_meta, items, progress_cb): + res_images_dir = data_yaml[set_name] + res_labels_dir = data_yaml[f"labels_{set_name}"] + classes_names = data_yaml["names"] - _process_split(project, data_yaml["names"], data_yaml["train"], data_yaml["labels_train"], train_split, progress_cb) - _process_split(project, data_yaml["names"], data_yaml["val"], data_yaml["labels_val"], val_split, progress_cb) + for batch in sly.batched(items, batch_size=max(int(len(items) / 50), 10)): + for item in batch: + ann = sly.Annotation.load_json_file(item.ann_path, project_meta) + save_ann_path = os.path.join(res_labels_dir, f"{sly.fs.get_file_name(item.name)}.txt") + _transform_annotation(ann, classes_names, save_ann_path) + save_img_path = os.path.join(res_images_dir, sly.fs.get_file_name_with_ext(item.img_path)) + sly.fs.hardlink_or_copy_file(item.img_path, save_img_path) # to speedup and save drive space + progress_cb(len(batch)) + + +def transform(sly_project_dir, yolov5_output_dir, train_set, val_set, progress_cb): + project = sly.Project(sly_project_dir, sly.OpenMode.READ) + data_yaml = _create_data_config(yolov5_output_dir, project.meta) + + _transform_set("train", data_yaml, project.meta, train_set, progress_cb) + _transform_set("val", data_yaml, project.meta, val_set, progress_cb) \ No newline at end of file diff --git a/train.py b/train.py index db810e03fefa..f3f5cd75edca 100644 --- a/train.py +++ b/train.py @@ -44,7 +44,10 @@ import supervisely_lib as sly from supervisely_lib import logger + def train(hyp, opt, device, tb_writer=None): + train_batches_uploaded = False + logger.info('hyperparameters', extra=hyp) save_dir, epochs, batch_size, total_batch_size, weights, rank = \ Path(opt.save_dir), opt.epochs, opt.batch_size, opt.total_batch_size, opt.weights, opt.global_rank @@ -207,7 +210,6 @@ def train(hyp, opt, device, tb_writer=None): hyp=hyp, cache=opt.cache_images and not opt.notest, rect=True, rank=-1, world_size=opt.world_size, workers=opt.workers, pad=0.5, prefix=colorstr('val: '))[0] - if not opt.resume: labels = np.concatenate(dataset.labels, 0) c = torch.tensor(labels[:, 0]) # classes @@ -215,6 +217,8 @@ def train(hyp, opt, device, tb_writer=None): # model._initialize_biases(cf.to(device)) if plots: plot_labels(labels, names, save_dir, loggers) + if opt.sly: + upload_label_vis() if tb_writer: tb_writer.add_histogram('classes', c, 0) @@ -353,13 +357,18 @@ def train(hyp, opt, device, tb_writer=None): # Plot if plots and ni < 3: f = save_dir / f'train_batch{ni}.jpg' # filename - Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start() + plot_images(imgs, targets, paths, f) + #Thread(target=plot_images, args=(imgs, targets, paths, f), daemon=True).start() # if tb_writer: # tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch) # tb_writer.add_graph(torch.jit.trace(model, imgs, strict=False), []) # add model graph elif plots and ni == 10 and wandb_logger.wandb: wandb_logger.log({"Mosaics": [wandb_logger.wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg') if x.exists()]}) + if plots and ni == 10 and opt.sly: + train_batches_uploaded = True + upload_train_data_vis() + # end batch ------------------------------------------------------------------------------------------------ # end epoch ---------------------------------------------------------------------------------------------------- @@ -386,7 +395,8 @@ def train(hyp, opt, device, tb_writer=None): plots=plots and final_epoch, wandb_logger=wandb_logger, compute_loss=compute_loss, - is_coco=is_coco) + is_coco=is_coco, + opt_sly=opt.sly) # Write with open(results_file, 'a') as f: @@ -413,7 +423,7 @@ def train(hyp, opt, device, tb_writer=None): metrics[tag] = x if opt.sly: - send_metrics(epoch, epochs, metrics) + send_metrics(epoch, epochs, metrics, opt.metrics_period) # Update best mAP fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] @@ -443,6 +453,11 @@ def train(hyp, opt, device, tb_writer=None): del ckpt # end epoch ---------------------------------------------------------------------------------------------------- + + if plots and opt.sly and train_batches_uploaded is False: + train_batches_uploaded = True + upload_train_data_vis() + # end training if rank in [-1, 0]: # Plots @@ -524,6 +539,8 @@ def main(): parser.add_argument('--save_period', type=int, default=-1, help='Log model after every "save_period" epoch') parser.add_argument('--artifact_alias', type=str, default="latest", help='version of dataset artifact to be used') parser.add_argument('--sly', action='store_true', help='for Supervisely App integration') + parser.add_argument('--metrics_period', type=int, default=1, help='Log metrics to Supervisely every "metrics_period" epochs') + opt = parser.parse_args() print("Input arguments:", opt)