diff --git a/core/storymanager/rank/rank.py b/core/storymanager/rank/rank.py index 403fa79e..342cae4a 100644 --- a/core/storymanager/rank/rank.py +++ b/core/storymanager/rank/rank.py @@ -196,7 +196,7 @@ def _get_selected(self, test_cases, test_results) -> pd.DataFrame: all_df = copy.deepcopy(self.all_df) selected_df = pd.DataFrame(all_df, columns=header) selected_df = selected_df.drop_duplicates(header[:-2]) - + # pylint: disable=E1136 paradigms = self.selected_dataitem.get("paradigms") if paradigms != ["all"]: selected_df = selected_df.loc[selected_df["paradigm"].isin(paradigms)] diff --git a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py index 6e226788..2e50ba7d 100644 --- a/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py +++ b/core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py @@ -24,7 +24,7 @@ from core.testcasecontroller.metrics import get_metric_func from core.common.utils import get_file_format, is_local_dir -os.environ['CUDA_VISIBLE_DEVICES'] = '0' +os.environ['CUDA_VISIBLE_DEVICES'] = '1' class LifelongLearning(ParadigmBase): # pylint: disable=too-many-locals @@ -162,6 +162,102 @@ def run(self): #BWT, FWT = self.compute(key, matrix) self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + elif mode == 'hard-example-mining': + dataset_files = self._split_dataset(splitting_dataset_times=rounds) + # pylint: disable=C0103 + # pylint: disable=C0206 + # pylint: disable=C0201 + # pylint: disable=W1203 + my_dict = {} + for r in range(rounds + 1): + train_dataset_file, eval_dataset_file = dataset_files[r] + if r == 0: + self.cloud_task_index = self._train(self.cloud_task_index, + train_dataset_file, + r) + + self.edge_task_index, tasks_detail, res = self.my_eval( + self.cloud_task_index, + eval_dataset_file, + r) + + else: + infer_dataset_file, eval_dataset_file = dataset_files[r] + inference_results, unseen_task_train_samples = self._inference( + self.cloud_task_index, + infer_dataset_file, + r) + samples_transfer_ratio_info.append((inference_results, + unseen_task_train_samples.x)) + + # If no unseen task samples in the this round, starting the next round + if len(unseen_task_train_samples.x) <= 0: + continue + + self.cloud_task_index = self._train(self.cloud_task_index, + unseen_task_train_samples, + r) + + tmp_dict = {} + for j in range(1, rounds+1): + _, eval_dataset_file = dataset_files[j] + self.edge_task_index, tasks_detail, res = self.my_eval( + self.cloud_task_index, + eval_dataset_file, + r) + LOGGER.info(f"train from round {r}") + LOGGER.info(f"test round {j}") + LOGGER.info(f"all scores: {res}") + score_list = tmp_dict.get("all", ['' for i in range(rounds)]) + score_list[j-1] = res + tmp_dict["all"] = score_list + task_avg_score = {'accuracy':0.0} + i = 0 + for detail in tasks_detail: + i += 1 + scores = detail.scores + entry = detail.entry + LOGGER.info(f"{entry} scores: {scores}") + task_avg_score['accuracy'] += scores['accuracy'] + score_list = tmp_dict.get(entry, ['' for i in range(rounds)]) + score_list[j-1] = scores + tmp_dict[entry] = score_list + task_avg_score['accuracy'] = task_avg_score['accuracy']/i + score_list = tmp_dict.get("task_avg", [{'accuracy':0.0} for i in range(rounds)]) + score_list[j-1] = task_avg_score + tmp_dict["task_avg"] = score_list + + for key in tmp_dict.keys(): + scores_list = my_dict.get(key, []) + scores_list.append(tmp_dict[key]) + my_dict[key] = scores_list + LOGGER.info(f"{key} scores: {scores_list}") + + + self.edge_task_index, tasks_detail, res = self.my_eval(self.cloud_task_index, + self.dataset.test_url, + rounds + 1) + task_avg_score = {'accuracy':0.0} + i = 0 + for detail in tasks_detail: + i += 1 + scores = detail.scores + entry = detail.entry + LOGGER.info(f"{entry} scores: {scores}") + task_avg_score['accuracy'] += scores['accuracy'] + task_avg_score['accuracy'] = task_avg_score['accuracy']/i + self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score + LOGGER.info(task_avg_score) + test_res, unseen_task_train_samples = self._inference(self.edge_task_index, + self.dataset.test_url, + "test") + for key in my_dict.keys(): + LOGGER.info(f"{key} scores: {my_dict[key]}") + for key in my_dict.keys(): + matrix = my_dict[key] + #BWT, FWT = self.compute(key, matrix) + self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix + elif mode != 'multi-inference': dataset_files = self._split_dataset(splitting_dataset_times=rounds) # pylint: disable=C0103 @@ -214,6 +310,7 @@ def _inference(self, edge_task_index, data_index_file, rounds): os.makedirs(unseen_task_saved_dir) os.environ["INFERENCE_RESULT_DIR"] = output_dir + os.environ["OUTPUT_URL"] = output_dir os.environ["MODEL_URLS"] = f"{edge_task_index}" inference_dataset = self.dataset.load_data(data_index_file, "eval", @@ -234,7 +331,7 @@ def _inference(self, edge_task_index, data_index_file, rounds): for i, _ in enumerate(inference_dataset.x): data = BaseDataSource(data_type="test") data.x = inference_dataset.x[i:(i + 1)] - res, is_unseen_task, _ = job.inference(data, **kwargs) + res, is_unseen_task, _ = job.inference_2(data, **kwargs) inference_results.append(res) if is_unseen_task: unseen_tasks.append(inference_dataset.x[i]) @@ -257,7 +354,7 @@ def _train(self, cloud_task_index, train_dataset, rounds): os.environ["CLOUD_KB_INDEX"] = cloud_task_index os.environ["OUTPUT_URL"] = train_output_dir - if rounds <= 1: + if rounds < 1: os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = 'False' else: os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = 'True' diff --git a/core/testenvmanager/dataset/dataset.py b/core/testenvmanager/dataset/dataset.py index 06843374..16bd038f 100644 --- a/core/testenvmanager/dataset/dataset.py +++ b/core/testenvmanager/dataset/dataset.py @@ -168,6 +168,12 @@ def split_dataset(self, dataset_url, dataset_format, ratio, method="default", output_dir=output_dir, times=times) + if method == "hard-example_splitting": + return self._hard_example_splitting(dataset_url, dataset_format, ratio, + data_types=dataset_types, + output_dir=output_dir, + times=times) + raise ValueError(f"dataset splitting method({method}) is not supported," f"currently, method supports 'default'.") @@ -320,6 +326,44 @@ def _city_splitting(self, data_file, data_format, ratio, return data_files + def _hard_example_splitting(self, data_file, data_format, ratio, + data_types=None, output_dir=None, times=1): + if not data_types: + data_types = ("train", "eval") + + if not output_dir: + output_dir = tempfile.mkdtemp() + + all_data = self._read_data_file(data_file, data_format) + + data_files = [] + + all_num = len(all_data) + step = int(all_num / (times*2)) + data_files.append(( + self._get_dataset_file(all_data[:int((all_num * ratio)/2)], output_dir, + data_types[0], 0, data_format), + self._get_dataset_file(all_data[int((all_num * ratio)/2):int(all_num/2)], output_dir, + data_types[1], 0, data_format))) + index = 1 + while index <= times: + if index == times: + new_dataset = all_data[int(all_num/2)+step*(index-1):] + else: + new_dataset = all_data[int(all_num/2)+step*(index-1): int(all_num/2)+step*index] + + new_num = len(new_dataset) + + data_files.append(( + self._get_dataset_file(new_dataset[:int(new_num * ratio)], output_dir, + data_types[0], index, data_format), + self._get_dataset_file(new_dataset[int(new_num * ratio):], output_dir, + data_types[1], index, data_format))) + + index += 1 + + return data_files + @classmethod def load_data(cls, file: str, data_type: str, label=None, use_raw=False, feature_process=None): """ diff --git a/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md new file mode 100644 index 00000000..d331d857 --- /dev/null +++ b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-SAM-based-Edge-Cloud-Collaboration.md @@ -0,0 +1,6 @@ +# Leaderboard of SAM-based Edge-Cloud Collaboration + + +| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | +| :--: | :-------------------------: | :----------------: | :----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :-----------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :--------------------------------: | ------------------- | +| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | \ No newline at end of file diff --git a/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md new file mode 100644 index 00000000..7691ef2c --- /dev/null +++ b/docs/leaderboards/leaderboard-in-semantic-segmentation-of-Cloud-Robotics/leaderboard-of-lifelong-learning.md @@ -0,0 +1,5 @@ +# Leaderboard of lifelong learning + +| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | +| :--: | :---------------------: | :----------------: | :-----------------: | :-----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :-----------------: | +| 1 | rfnet_lifelong_learning | 0.5206033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 10 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | diff --git a/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md b/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md index 51dd1bc3..02444f66 100644 --- a/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md +++ b/docs/proposals/algorithms/lifelong-learning/Real-Time IoT Perception Systems Based on Edge-Cloud Collaboration with Large Foundation Models.md @@ -63,7 +63,7 @@ The overall workflow of the system is as follows: ![plugin-workflow](images/plugin-workflow.jpg) -The Hard Example Mining Module will be implemented in the unknown sample recognition module of Ianvs. The Edge Inference Module will be implemented in the known sample inference module of Ianvs' edge-side knowledge management. The Cloud Inference Module will be implemented in the unknown task processing module of Ianvs' cloud-side knowledge management. +The Hard Example Mining Module will be implemented in the unknown sample recognition module of Ianvs. The Edge Inference Module will be implemented in the known sample inference module of Ianvs' edge-side knowledge management. The Cloud Inference Module will be implemented in the unknown sample inference module. The Liflelong Training module will be implemented in the unknown task processing module of Ianvs' cloud-side knowledge management. ### 3.2 Implementation of SAM-based Semantic Segmentation diff --git a/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg b/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg index 0638501d..8ffc28a9 100644 Binary files a/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg and b/docs/proposals/algorithms/lifelong-learning/images/joint-inference.jpg differ diff --git a/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg b/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg index 3ea7b40a..2241c5e4 100644 Binary files a/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg and b/docs/proposals/algorithms/lifelong-learning/images/plugin-workflow.jpg differ diff --git a/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl b/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl index 5fb53301..a1c7cf66 100644 Binary files a/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl and b/examples/resources/third_party/sedna-0.4.1-py3-none-any.whl differ diff --git a/examples/robot/lifelong_learning_bench/README.md b/examples/robot/lifelong_learning_bench/README.md index c113c14d..f8e9d6c2 100755 --- a/examples/robot/lifelong_learning_bench/README.md +++ b/examples/robot/lifelong_learning_bench/README.md @@ -1,27 +1,26 @@ # Quick Start -Welcome to Ianvs! Ianvs aims to test the performance of distributed synergy AI solutions following recognized standards, -in order to facilitate more efficient and effective development. Quick start helps you to test your algorithm on Ianvs -with a simple example of industrial defect detection. You can reduce manual procedures to just a few steps so that you can -build and start your distributed synergy AI solution development within minutes. +Welcome to Ianvs! Ianvs aims to test the performance of distributed synergy AI solutions following recognized standards, +in order to facilitate more efficient and effective development. Quick start helps you to test your algorithm on Ianvs +with a simple example of semantic segmentation based on lifelong learning. You can reduce manual procedures to just a few steps so that you can build and start your distributed synergy AI solution development within minutes. + +Before using Ianvs, you might want to have the device ready: -Before using Ianvs, you might want to have the device ready: - One machine is all you need, i.e., a laptop or a virtual machine is sufficient and a cluster is not necessary - 2 CPUs or more - 4GB+ free memory, depends on algorithm and simulation setting - 10GB+ free disk space - Internet connection for GitHub and pip, etc - Python 3.6+ installed - -In this example, we are using the Linux platform with Python 3.8.5. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. +In this example, we are using the Linux platform with Python 3.9. If you are using Windows, most steps should still apply but a few like commands and package requirements might be different. ## Step 1. Ianvs Preparation First, we download the code of Ianvs. Assuming that we are using `/ianvs` as workspace, Ianvs can be cloned with `Git` as: -``` shell +```shell mkdir /ianvs cd /ianvs #One might use another path preferred @@ -30,9 +29,9 @@ cd project git clone https://github.com/kubeedge/ianvs.git ``` +Then, we install third-party dependencies for ianvs. -Then, we install third-party dependencies for ianvs. -``` shell +```shell sudo apt-get update sudo apt-get install libgl1-mesa-glx -y python -m pip install --upgrade pip @@ -42,8 +41,9 @@ python -m pip install ./examples/resources/third_party/* python -m pip install -r requirements.txt ``` -We are now ready to install Ianvs. -``` shell +We are now ready to install Ianvs. + +```shell python setup.py install ``` @@ -52,14 +52,12 @@ python setup.py install Datasets and models can be large. To avoid over-size projects in the Github repository of Ianvs, the Ianvs code base does not include origin datasets. Then developers do not need to download non-necessary datasets for a quick start. -``` shell +```shell mkdir /data cd /data mkdir datasets cd datasets -python -m pip install kaggle -kaggle datasets download -d hsj576/cloud-robotics -unzip cloud-robotics.zip +download datasets in https://kubeedge-ianvs.github.io/download.html ``` The URL address of this dataset then should be filled in the configuration file ``testenv.yaml``. In this quick start, @@ -67,22 +65,70 @@ we have done that for you and the interested readers can refer to [testenv.yaml] +Related algorithm is also ready in this quick start. -Related algorithm is also ready in this quick start. -``` shell +```shell export PYTHONPATH=$PYTHONPATH:/ianvs/project/ianvs/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet ``` The URL address of this algorithm then should be filled in the configuration file ``algorithm.yaml``. In this quick start, we have done that for you and the interested readers can refer to [algorithm.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. +## Step 2.5. Large Vision Model Preparation(Optional) + +If you want to run the large vision model based cloud-edge collaboration process, then you need to follow the steps below to install the large vision model additionally. If you only want to run the basic lifelong learning process, you can ignore the steps below. + +In this example, we use [SAM model](https://segment-anything.com/) as the cloud large vision model. So, we need to install SAM by the following instructions: + +```bash +cd /ianvs/project +git clone https://github.com/facebookresearch/segment-anything.git +cd segment-anything +python -m pip install -e . +``` + +Then, we need to download the pretrained SAM model: + +```bash +wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth +``` + +In order to save the inference result, we need to install mmcv and mmdetection by the following instructions: + +```bash +python -m pip install https://download.openmmlab.com/mmcv/dist/cu118/torch2.0.0/mmcv-2.0.0-cp39-cp39-manylinux1_x86_64.whl +cd /ianvs/project +git clone https://github.com/hsj576/mmdetection.git +cd mmdetection +python -m pip install -v -e . +``` + +In case that your computer couldn't run SAM model, we prepare a cache for all the SAM inference results in Cloud-Robotics dataset. You could download the cache from [this link](https://pan.baidu.com/s/1oGGBa8TjZn0ccbznQsl48g?pwd=wpp1) and put the cache file in "/ianvs/project/": + +```bash +cp cache.pickle /ianvs/project +``` + + By using the cache, you could simulate the edge-cloud joint inference without installing SAM model. + +Besides that, we also provided you a pretrained RFNet model in [this link](https://pan.baidu.com/s/1h8JnUgr1hfx5QnaFLLkMAg?pwd=jts4), you could use it if you don't want to train the RFNet model from zero. This instruction is optional: + +```bash +cd /ianvs/project +mkdir pretrain +cp pretrain_model.pth /ianvs/project/pretrain +in /ianvs/project/ianvs/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/args.py set self.resume = '/ianvs/project/pretrain/pretrain_model.pth' +``` + ## Step 3. Ianvs Execution and Presentation -We are now ready to run the ianvs for benchmarking. +We are now ready to run the ianvs for benchmarking. + +To run the basic lifelong learning process: -``` shell +```shell cd /ianvs/project/ianvs -ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob.yaml +ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml ``` Finally, the user can check the result of benchmarking on the console and also in the output path( @@ -90,19 +136,34 @@ e.g. `/ianvs/lifelong_learning_bench/workspace`) defined in the benchmarking con e.g. `benchmarkingjob.yaml`). In this quick start, we have done all configurations for you and the interested readers can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. -The final output might look like this: +The final output might look like this: + +| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | url | +| :--: | :---------------------: | :----------------: | :-----------------: | :-----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :-----------------: | :-----------------------------------------------------------------------------------------------------------------------------: | +| 1 | rfnet_lifelong_learning | 0.2970033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | /ianvs/lifelong_learning_bench/robot-workspace-bwt/benchmarkingjob/rfnet_lifelong_learning/efdc47a2-f9fb-11ed-8f8b-0242ac110007 | +To run the large vision model based cloud-edge collaboration process: -| rank | algorithm | accuracy | BWT | FWT | paradigm | basemodel | task_definition | task_allocation | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | time | url | -|:------:|:-------------------------:|:--------------------:|:---------------------:|:---------------------:|:------------------:|:-----------:|:------------------------:|:------------------------:|:-------------------------:|:------------------:|:-------------------------:|:-------------------------:|:---------------------:|:---------------------------------------------------------------------------------------------------------------------------------:| -| 1 | rfnet_lifelong_learning | 0.2970033189775575 | 0.04239649121511442 | 0.02299711942108413 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 2023-05-24 15:07:57 | /ianvs/lifelong_learning_bench/robot-workspace-bwt/benchmarkingjob/rfnet_lifelong_learning/efdc47a2-f9fb-11ed-8f8b-0242ac110007 | +```shell +cd /ianvs/project/ianvs +ianvs -f examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml +``` + +Finally, the user can check the result of benchmarking on the console and also in the output path( +e.g. `/ianvs/lifelong_learning_bench/workspace`) defined in the benchmarking config file ( +e.g. `benchmarkingjob.yaml`). In this quick start, we have done all configurations for you and the interested readers +can refer to [benchmarkingJob.yaml](https://ianvs.readthedocs.io/en/latest/guides/how-to-test-algorithms.html#step-1-test-environment-preparation) for more details. +The final output might look like this: +| rank | algorithm | accuracy | Task_Avg_Acc | paradigm | basemodel | task_definition | task_allocation | unseen_sample_recognition | basemodel-learning_rate | basemodel-epochs | task_definition-origins | task_allocation-origins | unseen_sample_recognition-threhold | time | url | +| :--: | :-------------------------: | :----------------: | :----------------: | :--------------: | :-------: | :--------------------: | :--------------------: | :-----------------------: | :---------------------: | :--------------: | :---------------------: | :---------------------: | :--------------------------------: | ------------------- | :---------------------------------------------------------------------------------------------------------------: | +| 1 | sam_rfnet_lifelong_learning | 0.7052917006987501 | 0.6258875117354328 | lifelonglearning | BaseModel | TaskDefinitionByOrigin | TaskAllocationByOrigin | HardSampleMining | 0.0001 | 1 | ['front', 'garden'] | ['front', 'garden'] | 0.95 | 2023-08-24 12:43:19 | /ianvs/sam_bench/robot-workspace/benchmarkingjob/sam_rfnet_lifelong_learning/9465c47a-4235-11ee-8519-ec2a724ccd3e | This ends the quick start experiment. # What is next -If any problems happen, the user can refer to [the issue page on Github](https://github.com/kubeedge/ianvs/issues) for help and are also welcome to raise any new issue. +If any problems happen, the user can refer to [the issue page on Github](https://github.com/kubeedge/ianvs/issues) for help and are also welcome to raise any new issue. -Enjoy your journey on Ianvs! \ No newline at end of file +Enjoy your journey on Ianvs! diff --git a/examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml b/examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml new file mode 100644 index 00000000..01cb80c2 --- /dev/null +++ b/examples/robot/lifelong_learning_bench/benchmarkingjob-sam.yaml @@ -0,0 +1,82 @@ +benchmarkingjob: + # job name of bechmarking; string type; + name: "benchmarkingjob" + # the url address of job workspace that will reserve the output of tests; string type; + workspace: "/ianvs/sam_bench/robot-workspace" + #workspace: "/home/hsj/ianvs/sam_bench/cloud-robot-workspace" + + # the url address of test environment configuration file; string type; + # the file format supports yaml/yml; + testenv: "./examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml" + #testenv: "./examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml" + + # the configuration of test object + test_object: + # test type; string type; + # currently the option of value is "algorithms",the others will be added in succession. + type: "algorithms" + # test algorithm configuration files; list type; + algorithms: + # algorithm name; string type; + #- name: "rfnet_lifelong_learning" + - name: "sam_rfnet_lifelong_learning" + #- name: "vit_lifelong_learning" + #- name: "sam_vit_lifelong_learning" + # the url address of test algorithm configuration file; string type; + # the file format supports yaml/yml + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml" + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml" + #url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml" + # the url address of test algorithm configuration file; string type; + # the file format supports yaml/yml + + # the configuration of ranking leaderboard + rank: + # rank leaderboard with metric of test case's evaluation and order ; list type; + # the sorting priority is based on the sequence of metrics in the list from front to back; + sort_by: [ { "accuracy": "descend" } ] + + # visualization configuration + visualization: + # mode of visualization in the leaderboard; string type; + # There are quite a few possible dataitems in the leaderboard. Not all of them can be shown simultaneously on the screen. + # In the leaderboard, we provide the "selected_only" mode for the user to configure what is shown or is not shown. + mode: "selected_only" + # method of visualization for selected dataitems; string type; + # currently the options of value are as follows: + # 1> "print_table": print selected dataitems; + method: "print_table" + + # selected dataitem configuration + # The user can add his/her interested dataitems in terms of "paradigms", "modules", "hyperparameters" and "metrics", + # so that the selected columns will be shown. + selected_dataitem: + # currently the options of value are as follows: + # 1> "all": select all paradigms in the leaderboard; + # 2> paradigms in the leaderboard, e.g., "singletasklearning" + paradigms: [ "all" ] + # currently the options of value are as follows: + # 1> "all": select all modules in the leaderboard; + # 2> modules in the leaderboard, e.g., "basemodel" + modules: [ "all" ] + # currently the options of value are as follows: + # 1> "all": select all hyperparameters in the leaderboard; + # 2> hyperparameters in the leaderboard, e.g., "momentum" + hyperparameters: [ "all" ] + # currently the options of value are as follows: + # 1> "all": select all metrics in the leaderboard; + # 2> metrics in the leaderboard, e.g., "F1_SCORE" + metrics: [ "accuracy", "Task_Avg_Acc"] + + # model of save selected and all dataitems in workspace `./rank` ; string type; + # currently the options of value are as follows: + # 1> "selected_and_all": save selected and all dataitems; + # 2> "selected_only": save selected dataitems; + save_mode: "selected_and_all_and_picture" + + + + + + diff --git a/examples/robot/lifelong_learning_bench/benchmarkingjob.yaml b/examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml old mode 100755 new mode 100644 similarity index 97% rename from examples/robot/lifelong_learning_bench/benchmarkingjob.yaml rename to examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml index 3e6b5a49..24f795a3 --- a/examples/robot/lifelong_learning_bench/benchmarkingjob.yaml +++ b/examples/robot/lifelong_learning_bench/benchmarkingjob-simple.yaml @@ -2,7 +2,7 @@ benchmarkingjob: # job name of bechmarking; string type; name: "benchmarkingjob" # the url address of job workspace that will reserve the output of tests; string type; - workspace: "/home/vipuser/ianvs/lifelong_learning_bench/robot-workspace-test" + workspace: "/ianvs/lifelong_learning_bench/robot-workspace-test" # the url address of test environment configuration file; string type; # the file format supports yaml/yml; diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py new file mode 100644 index 00000000..c8ad1d27 --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/cityscapes_id2label.py @@ -0,0 +1,34 @@ +CONFIG = { +"id2label": { + "0": "road", + "1": "sidewalk", + "2": "building", + "3": "wall", + "4": "fence", + "5": "pole", + "6": "traffic light", + "7": "traffic sign", + "8": "vegetation", + "9": "terrain", + "10": "sky", + "11": "person", + "12": "rider", + "13": "car", + "14": "truck", + "15": "bus", + "16": "train", + "17": "motorcycle", + "18": "bicycle", + "19": "stair", + "20": "curb", + "21": "ramp", + "22": "runway", + "23": "flowerbed", + "24": "door", + "25": "CCTV camera", + "26": "Manhole", + "27": "hydrant", + "28": "belt", + "29": "dustbin" + }, +} \ No newline at end of file diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py index 2d303429..da3538b8 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/eval.py @@ -7,6 +7,12 @@ from torchvision.transforms import ToPILImage from PIL import Image +from segment_anything import sam_model_registry, SamAutomaticMaskGenerator +import mmcv +import pycocotools.mask as maskUtils +from mmdet.visualization.image import imshow_det_bboxes +import pickle + from dataloaders import make_data_loader from dataloaders.utils import decode_seg_map_sequence, Colorize from utils.metrics import Evaluator @@ -15,6 +21,9 @@ from models.resnet.resnet_single_scale_single_attention import * from models.resnet import resnet_single_scale_single_attention_unseen import torch.backends.cudnn as cudnn +from cityscapes_id2label import CONFIG as CONFIG_CITYSCAPES_ID2LABEL +import torch.nn.functional as F +from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation os.environ["OMP_NUM_THREADS"] = "1" os.environ["MKL_NUM_THREADS"] = "1" @@ -54,12 +63,257 @@ def __init__(self, args, data=None, unseen_detection=False): # self.new_state_dict = torch.load(args.weight_path, map_location=torch.device("cpu")) # self.model = load_my_state_dict(self.model, self.new_state_dict['state_dict']) + def segformer_segmentation(self, image, processor, model, rank): + h, w, _ = image.shape + inputs = processor(images=image, return_tensors="pt").to(rank) + outputs = model(**inputs) + logits = outputs.logits + logits = F.interpolate(logits, size=(h, w), mode='bilinear', align_corners=True) + predicted_semantic_map = logits.argmax(dim=1) + return predicted_semantic_map + + def draw_mask(self, image_name, mask, output_path): + img = mmcv.imread(image_name) + anns = {'annotations': mask} + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + semantc_mask = torch.zeros(1024, 2048) + i = 0 + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + semantc_mask[valid_mask] = i + i += 1 + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = str(i) + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + '_mask.png')) + print('[Save] save mask: ', os.path.join(output_path, filename + '_mask.png')) + semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def draw_picture(self, image_name, semantc_mask, id2label, output_path, suffix): + img = mmcv.imread(image_name) + sematic_class_in_img = torch.unique(semantc_mask) + semantic_bitmasks, semantic_class_names = [], [] + + # semantic prediction + for i in range(len(sematic_class_in_img)): + class_name = id2label['id2label'][str(sematic_class_in_img[i].item())] + class_mask = semantc_mask == sematic_class_in_img[i] + class_mask = class_mask.cpu().numpy().astype(np.uint8) + semantic_class_names.append(class_name) + semantic_bitmasks.append(class_mask) + + #print(os.environ["OUTPUT_URL"]) + length = len(image_name) + for i in range(length): + if image_name[length-i-1] == '_': + break + filename = image_name[length-i:] + imshow_det_bboxes(img, + bboxes=None, + labels=np.arange(len(sematic_class_in_img)), + segms=np.stack(semantic_bitmasks), + class_names=semantic_class_names, + font_size=25, + show=False, + out_file=os.path.join(output_path, filename + suffix)) + print('[Save] save rfnet prediction: ', os.path.join(output_path, filename + suffix)) + #semantc_mask = semantc_mask.unsqueeze(0).numpy() + del img + del semantic_bitmasks + del semantic_class_names + + def confidence(self, input_output): + output = torch.softmax(input_output, dim=0) + highth = len(output[0]) + width = len(output[0][0]) + sum_1 = 0.0 + sum_2 = 0.0 + values, _ = torch.topk(output, k=2, dim=0) + sum_1 = torch.sum(values[0]) + value_2 = torch.sub(values[0],values[1]) + sum_2 = torch.sum(value_2) + sum_1 = sum_1/(highth*width) + sum_2 = sum_2/(highth*width) + count = (values[0] > 0.9).sum().item() + sum_3 = count/(highth*width) + return sum_3 + + def sam_predict_ssa(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + semantc_mask = pred.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + propose_classes_ids = pred[valid_mask] + num_class_proposals = len(torch.unique(propose_classes_ids)) + if num_class_proposals == 1: + semantc_mask[valid_mask] = propose_classes_ids[0] + ann['class_name'] = id2label['id2label'][str(propose_classes_ids[0].item())] + ann['class_proposals'] = id2label['id2label'][str(propose_classes_ids[0].item())] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + continue + top_1_propose_class_ids = torch.bincount(propose_classes_ids.flatten()).topk(1).indices + top_1_propose_class_names = [id2label['id2label'][str(class_id.item())] for class_id in top_1_propose_class_ids] + #print("top_1_propose_class_ids: ", top_1_propose_class_ids) + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names[0] + ann['class_proposals'] = top_1_propose_class_names[0] + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del num_class_proposals + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + + def sam_predict(self, image_name, pred): + with open('/home/hsj/ianvs/project/cache.pickle', 'rb') as file: + cache = pickle.load(file) + img = mmcv.imread(image_name) + if image_name in cache.keys(): + mask = cache[image_name] + print("load cache") + else: + sam = sam_model_registry["vit_h"](checkpoint="/home/hsj/ianvs/project/segment-anything/sam_vit_h_4b8939.pth").to('cuda:1') + mask_branch_model = SamAutomaticMaskGenerator( + model=sam, + #points_per_side=64, + # Foggy driving (zero-shot evaluate) is more challenging than other dataset, so we use a larger points_per_side + #pred_iou_thresh=0.86, + #stability_score_thresh=0.92, + #crop_n_layers=1, + #crop_n_points_downscale_factor=2, + #min_mask_region_area=100, # Requires open-cv to run post-processing + output_mode='coco_rle', + ) + print('[Model loaded] Mask branch (SAM) is loaded.') + mask = mask_branch_model.generate(img) + cache[image_name] = mask + with open('/home/hsj/ianvs/project/cache.pickle', 'wb') as file: + pickle.dump(cache, file) + print("save cache") + + anns = {'annotations': mask} + #print(len(anns['annotations']), len(anns['annotations'][0])) + #print(pred.shape) + #print(pred[0]) + class_names = [] + pred_2 = np.argmax(pred, axis=0) + semantc_mask = pred_2.clone() + id2label = CONFIG_CITYSCAPES_ID2LABEL + anns['annotations'] = sorted(anns['annotations'], key=lambda x: x['area'], reverse=True) + for ann in anns['annotations']: + valid_mask = torch.tensor(maskUtils.decode(ann['segmentation'])).bool() + #print(valid_mask) + length = len(pred) + all_scores = [0 for i in range(length)] + for i in range(length): + propose_classes_ids = pred[i][valid_mask] + #print(propose_classes_ids.shape) + all_scores[i] = torch.sum(propose_classes_ids) + #print(all_scores[i]) + top_1_propose_class_ids = np.argmax(all_scores) + #print(top_1_propose_class_ids) + top_1_propose_class_names = id2label['id2label'][str(top_1_propose_class_ids)] + + semantc_mask[valid_mask] = top_1_propose_class_ids + ann['class_name'] = top_1_propose_class_names + ann['class_proposals'] = top_1_propose_class_names + class_names.append(ann['class_name']) + # bitmasks.append(maskUtils.decode(ann['segmentation'])) + + del valid_mask + del propose_classes_ids + del top_1_propose_class_ids + del top_1_propose_class_names + + #print(semantc_mask.shape) + #print(semantc_mask) + + #self.draw_picture(img, image_name, pred_2, id2label, output_path, "_origin.png") + #self.draw_picture(img, image_name, semantc_mask, id2label, output_path, "_sam.png") + + del img + del anns + #del semantc_mask + # del bitmasks + del class_names + return semantc_mask, mask + def validate(self): #print("start validating 55") self.model.eval() self.evaluator.reset() tbar = tqdm(self.test_loader, desc='\r') predictions = [] + scores = [] for i, (sample, image_name) in enumerate(tbar):#self.test_loader: if self.args.depth: image, depth, target = sample['image'], sample['depth'], sample['label'] @@ -81,46 +335,131 @@ def validate(self): if self.args.cuda: torch.cuda.synchronize() + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + pred = output.data.cpu().numpy() # todo pred = np.argmax(pred, axis=1) predictions.append(pred) - if not self.args.save_predicted_image: - continue + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_origin.png") - pre_colors = Colorize()(torch.max(output, 1)[1].detach().cpu().byte()) - pre_labels = torch.max(output, 1)[1].detach().cpu().byte() - print(pre_labels.shape) - # save - for i in range(pre_colors.shape[0]): - print(image_name[0]) + #print("start validating 120") + return predictions, scores + + def vit_validate(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + predictions.append(pred) + + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + + #print("start validating 120") + return predictions - if not image_name[0]: - img_name = "test.png" + def validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + scores = [] + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + if self.args.depth: + image, depth, target = sample['image'], sample['depth'], sample['label'] + else: + # spec = time.time() + image, target = sample['image'], sample['label'] + #print(self.args.cuda, self.args.gpu_ids) + if self.args.cuda: + image = image.cuda(self.args.gpu_ids) + if self.args.depth: + depth = depth.cuda(self.args.gpu_ids) + + with torch.no_grad(): + if self.args.depth: + output = self.model(image, depth) else: - img_name = os.path.basename(image_name[0]) + output = self.model(image) + + if self.args.cuda: + torch.cuda.synchronize() - color_label_name = os.path.join(self.args.color_label_save_path, img_name) - label_name = os.path.join(self.args.label_save_path, img_name) - merge_label_name = os.path.join(self.args.merge_label_save_path, img_name) + if len(output) == 1: + score = self.confidence(output[0]) + else: + score = self.confidence(output) + scores.append(score) + + pred = output.data.cpu().numpy() + # todo + pred_sam, mask = self.sam_predict(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) - os.makedirs(os.path.dirname(color_label_name), exist_ok=True) - os.makedirs(os.path.dirname(merge_label_name), exist_ok=True) - os.makedirs(os.path.dirname(label_name), exist_ok=True) + predictions.append(np.array(pred_sam)) - pre_color_image = ToPILImage()(pre_colors[i]) # pre_colors.dtype = float64 - pre_color_image.save(color_label_name) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_sam.png") + self.draw_mask(image_name[0], mask, output_path) + + #print("start validating 120") + return predictions, scores - pre_label_image = ToPILImage()(pre_labels[i]) - pre_label_image.save(label_name) + def vit_validate_cloud(self): + #print("start validating 55") + self.model.eval() + self.evaluator.reset() + tbar = tqdm(self.test_loader, desc='\r') + predictions = [] + rank = 'cuda:0' + semantic_branch_processor = SegformerFeatureExtractor.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024") + semantic_branch_model = SegformerForSemanticSegmentation.from_pretrained( + "nvidia/segformer-b5-finetuned-cityscapes-1024-1024").to(rank) + for i, (sample, image_name) in enumerate(tbar):#self.test_loader: + img = mmcv.imread(image_name[0]) + class_ids = self.segformer_segmentation(img, semantic_branch_processor, semantic_branch_model, rank) + pred = class_ids.data.cpu().numpy() + pred_sam, mask = self.sam_predict_ssa(image_name[0], torch.from_numpy(pred[0])) + if pred_sam.ndim < 3: + h, w = pred_sam.shape + pred_sam = pred_sam.reshape(1, h, w) + #print(pred_sam.shape) + predictions.append(np.array(pred_sam)) - if (self.args.merge): - image_merge(image[i], pre_color_image, merge_label_name) - print('save image: {}'.format(merge_label_name)) + output_path = os.environ["OUTPUT_URL"] + id2label = CONFIG_CITYSCAPES_ID2LABEL + self.draw_picture(image_name[0], torch.from_numpy(pred[0]), id2label, output_path, "_vit_origin.png") + self.draw_picture(image_name[0], pred_sam[0], id2label, output_path, "_vit_sam.png") + self.draw_mask(image_name[0], mask, output_path) + #print("start validating 120") return predictions - + def task_divide(self): seen_task_samples, unseen_task_samples = [], [] self.model.eval() diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py index b2e93683..0acef918 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/RFNet/utils/metrics.py @@ -188,6 +188,8 @@ def add_batch(self, gt_image, pre_image): gt_image = np.array(gt_image) pre_image = np.array(pre_image) print(gt_image.shape, pre_image.shape) + if gt_image.shape != pre_image.shape: + pre_image = pre_image[0] if gt_image.shape != pre_image.shape: pre_image = pre_image[0] assert gt_image.shape == pre_image.shape diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py new file mode 100644 index 00000000..056627b8 --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py @@ -0,0 +1,180 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return predictions + + def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions = self.validator.validate_cloud() + return predictions + + def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py index c2ad091e..4684d567 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py @@ -40,16 +40,6 @@ def get_weights(self): def set_weights(self, weights): self.trainer.set_weight(weights) - - epoch_num = 0 - print("Total epoch: ", epoch_num) - loss_all = [] - for epoch in range(epoch_num): - train_loss = self.trainer.my_training(epoch) - loss_all.append(train_loss) - with open('/home/shijing.hu/ianvs/project/ianvs/train_loss_2.txt', 'a+') as file: - np.savetxt(file, loss_all) - file.close def train(self, train_data, valid_data=None, **kwargs): self.trainer = Trainer(self.train_args, train_data=train_data) @@ -82,27 +72,60 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return predictions + + def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return predictions + + def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ if len(data) > 10: - print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) else: - print("predict start for small data") if not isinstance(data[0][0], dict): data = self._preprocess(data) - #print("predict starting 69") if type(data) is np.ndarray: data = data.tolist() - #print("predict starting 72") - #print("predict starting 73") self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, pin_memory=True) - #print("predict starting 75") - return self.validator.validate() + predictions, scores = self.validator.validate() + return scores def evaluate(self, data, **kwargs): - #print("evaluate starting 77") self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) samples = self._preprocess(data.x) predictions = self.predict(samples) diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py new file mode 100644 index 00000000..cfde33e6 --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py @@ -0,0 +1,180 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + """ + Use the Segformer model to predict at the edge + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions = self.validator.vit_validate() + return predictions + + def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions = self.validator.vit_validate_cloud() + return predictions + + def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py new file mode 100644 index 00000000..c83b7b1e --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py @@ -0,0 +1,180 @@ +import os +import gc +import numpy as np +import torch +from torch.utils.data import DataLoader +from sedna.common.class_factory import ClassType, ClassFactory +from sedna.common.config import Context +from sedna.common.file_ops import FileOps +from sedna.common.log import LOGGER +from PIL import Image +from torchvision import transforms + +from RFNet.train import Trainer +from RFNet.eval import Validator, load_my_state_dict +from RFNet.dataloaders import custom_transforms as tr +from RFNet.dataloaders import make_data_loader +from RFNet.utils.args import TrainArgs, ValArgs + +# set backend +os.environ['BACKEND_TYPE'] = 'PYTORCH' + +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +@ClassFactory.register(ClassType.GENERAL, alias="BaseModel") +class BaseModel: + def __init__(self, **kwargs): + self.train_args = TrainArgs(**kwargs) + self.trainer = None + + self.val_args = ValArgs(**kwargs) + label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results") + self.val_args.color_label_save_path = os.path.join(label_save_dir, "color") + self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge") + self.val_args.label_save_path = os.path.join(label_save_dir, "label") + self.validator = Validator(self.val_args) + + def get_weights(self): + return self.trainer.get_weight() + + def set_weights(self, weights): + self.trainer.set_weight(weights) + + def train(self, train_data, valid_data=None, **kwargs): + self.trainer = Trainer(self.train_args, train_data=train_data) + print("Total epoches:", self.trainer.args.epochs) + loss_all = [] + for epoch in range( + self.trainer.args.start_epoch, + self.trainer.args.epochs): + if epoch == 0 and self.trainer.val_loader: + self.trainer.validation(epoch) + loss = self.trainer.training(epoch) + loss_all.append(loss) + if self.trainer.args.no_val and ( + epoch % + self.trainer.args.eval_interval == ( + self.trainer.args.eval_interval - + 1) or epoch == self.trainer.args.epochs - + 1): + # save checkpoint when it meets eval_interval or the training + # finished + is_best = False + self.train_model_url = self.trainer.saver.save_checkpoint({ + 'epoch': epoch + 1, + 'state_dict': self.trainer.model.state_dict(), + 'optimizer': self.trainer.optimizer.state_dict(), + 'best_pred': self.trainer.best_pred, + }, is_best) + + self.trainer.writer.close() + return self.train_model_url + + def predict(self, data, **kwargs): + """ + Use the Segformer model to predict at the edge + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions = self.validator.vit_validate() + return predictions + + def predict_cloud(self, data, **kwargs): + """ + Use the SAM model to predict at the cloud + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions = self.validator.vit_validate() + return predictions + + def predict_score(self, data, **kwargs): + """ + Get the prediction scores of RFNet model + """ + if len(data) > 10: + my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} + _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs) + else: + if not isinstance(data[0][0], dict): + data = self._preprocess(data) + if type(data) is np.ndarray: + data = data.tolist() + self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False, + pin_memory=True) + + predictions, scores = self.validator.validate() + return scores + + def evaluate(self, data, **kwargs): + self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True) + samples = self._preprocess(data.x) + predictions = self.predict(samples) + metric_name, metric_func = kwargs.get("metric") + if callable(metric_func): + return metric_func(data.y, predictions) + else: + raise Exception(f"not found model metric func(name={metric_name}) in model eval phase") + + def load(self, model_url, **kwargs): + if model_url: + print("load model url: ",model_url) + self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu")) + self.train_args.resume = model_url + else: + raise Exception("model url does not exist.") + self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict']) + + def save(self, model_path=None): + # TODO: save unstructured data model + if not model_path: + LOGGER.warning(f"Not specify model path.") + return self.train_model_url + + return FileOps.upload(self.train_model_url, model_path) + + def _preprocess(self, image_urls): + transformed_images = [] + for paths in image_urls: + if len(paths) == 2: + img_path, depth_path = paths + _img = Image.open(img_path).convert('RGB') + _depth = Image.open(depth_path) + #print(img_path, depth_path) + else: + img_path = paths[0] + #print(img_path) + _img = Image.open(img_path).convert('RGB') + _depth = _img + + sample = {'image': _img, 'depth': _depth, 'label': _img} + del _img + gc.collect() + composed_transforms = transforms.Compose([ + # tr.CropBlackArea(), + # tr.FixedResize(size=self.args.crop_size), + tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + tr.ToTensor()]) + + transformed_images.append((composed_transforms(sample), img_path)) + + return transformed_images diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py index 37e3fbd9..b93e2f48 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py @@ -89,6 +89,9 @@ def train(self, train_data, valid_data=None, **kwargs): return self.train_model_url def predict(self, data, **kwargs): + """ + Use the RFNet model to predict at the edge + """ if len(data) > 10: print("predict start for big data") my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True} diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py new file mode 100644 index 00000000..ab7772ce --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py @@ -0,0 +1,56 @@ +# @inproceedings{zhou2021learning, +# author = {Zhou, Da-Wei and Ye, Han-Jia and Zhan, De-Chuan}, +# title = {Learning Placeholders for Open-Set Recognition}, +# booktitle = {CVPR}, +# pages = {4401-4410}, +# year = {2021} +# } +from typing import List, Any, Tuple + +from sedna.datasources import BaseDataSource +from sedna.common.class_factory import ClassFactory, ClassType +import random + +__all__ = ('HardSampleMining',) + +@ClassFactory.register(ClassType.UTD, alias="HardSampleMining") +class HardSampleMining: + """ + Dividing the data set based on whether the sample is a known class of data. + + Parameters + ---------- + model_path: string + Path of the model + """ + def __init__(self, **kwargs): + self.threhold = kwargs.get("threhold") + + def __call__(self, + samples: BaseDataSource, scores, **kwargs) -> Tuple[BaseDataSource, + BaseDataSource]: + ''' + Parameters + ---------- + samples : BaseDataSource + inference samples + + Returns + ------- + seen_task_samples: BaseDataSource + unseen_task_samples: BaseDataSource + ''' + seen_task_samples = BaseDataSource(data_type=samples.data_type) + unseen_task_samples = BaseDataSource(data_type=samples.data_type) + + if scores[0] > self.threhold: + print(f"found easy sample, confidence score: {scores[0]}") + seen_task_samples.x = samples.x + unseen_task_samples.x = [] + + else: + print(f"found hard sample, confidence score: {scores[0]}") + seen_task_samples.x = [] + unseen_task_samples.x = samples.x + + return seen_task_samples, unseen_task_samples diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml index 628ceb9a..6d7de7f7 100755 --- a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/rfnet_algorithm.yaml @@ -12,8 +12,9 @@ algorithm: # the method of splitting dataset; string type; optional; # currently the options of value are as follows: # 1> "default": the dataset is evenly divided based train_ratio; - splitting_method: "default" - #splitting_method: "my_splitting" + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" # algorithm module configuration in the paradigm; list type; modules: @@ -25,7 +26,7 @@ algorithm: # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; name: "BaseModel" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; @@ -34,30 +35,40 @@ algorithm: - 0.0001 - epochs: values: - - 1 + - 10 # 2> "task_definition": define lifelong task ; optional module; - type: "task_definition" # name of python module; string type; name: "TaskDefinitionByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] + - [ "front", "garden" ] # 3> "task_allocation": allocate lifelong task ; optional module; - type: "task_allocation" # name of python module; string type; name: "TaskAllocationByOrigin" # the url address of python module; string type; - url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin.py" + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" # hyperparameters configuration for the python module; list type; hyperparameters: # name of the hyperparameter; string type; # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. - origins: values: - - [ "real", "sim" ] \ No newline at end of file + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml new file mode 100644 index 00000000..89715d9f --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-sam.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml new file mode 100644 index 00000000..9fec28de --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/sam_vit_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-sam.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml new file mode 100644 index 00000000..adff5710 --- /dev/null +++ b/examples/robot/lifelong_learning_bench/testalgorithms/rfnet/vit_algorithm.yaml @@ -0,0 +1,74 @@ +algorithm: + # paradigm type; string type; + # currently the options of value are as follows: + # 1> "singletasklearning" + # 2> "incrementallearning" + # 3> "lifelonglearning" + paradigm_type: "lifelonglearning" + lifelong_learning_data_setting: + # ratio of training dataset; float type; + # the default value is 0.8. + train_ratio: 0.9 + # the method of splitting dataset; string type; optional; + # currently the options of value are as follows: + # 1> "default": the dataset is evenly divided based train_ratio; + #splitting_method: "default" + #splitting_method: "fwt_splitting" + splitting_method: "hard-example_splitting" + + # algorithm module configuration in the paradigm; list type; + modules: + # type of algorithm module; string type; + # currently the options of value are as follows: + # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module; + - type: "basemodel" + # name of python module; string type; + # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; + name: "BaseModel" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/basemodel-vit-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + - learning_rate: + values: + - 0.0001 + - epochs: + values: + - 1 + # 2> "task_definition": define lifelong task ; optional module; + - type: "task_definition" + # name of python module; string type; + name: "TaskDefinitionByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_definition_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + # 3> "task_allocation": allocate lifelong task ; optional module; + - type: "task_allocation" + # name of python module; string type; + name: "TaskAllocationByOrigin" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/task_allocation_by_origin-simple.py" + # hyperparameters configuration for the python module; list type; + hyperparameters: + # name of the hyperparameter; string type; + # origins of data; value is ["real", "sim"], this means that data from real camera and simulator. + - origins: + values: + - [ "front", "garden" ] + + - type: "unseen_sample_recognition" + # name of python module; string type; + name: "HardSampleMining" + # the url address of python module; string type; + url: "./examples/robot/lifelong_learning_bench/testalgorithms/rfnet/hard_sample_mining.py" + hyperparameters: + - threhold: + values: + - 0.95 diff --git a/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml b/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml index 092abcb8..cde00454 100755 --- a/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml +++ b/examples/robot/lifelong_learning_bench/testenv/testenv-robot-small.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int diff --git a/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml b/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml index d224c3e6..e8645110 100755 --- a/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml +++ b/examples/robot/lifelong_learning_bench/testenv/testenv-robot.yaml @@ -14,7 +14,8 @@ testenv: name: "accuracy" # the url address of python file url: "./examples/robot/lifelong_learning_bench/testenv/accuracy.py" - mode: "no-inference" + #mode: "no-inference" + mode: "hard-example-mining" # condition of triggering inference model to update # threshold of the condition; types are float/int