[Feature] Add DMCP and fix the deploy pipeline of NAS algorithms

configs/_base_/settings/imagenet_bs2048_dmcp.py

@@ -0,0 +1,98 @@
+# dataset settings
+dataset_type = 'mmcls.ImageNet'
+
+max_search_epochs = 100
+# learning rate setting
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=0.5,
+        by_epoch=True,
+        begin=0,
+        end=10,
+        convert_to_iter_based=True),
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_search_epochs,
+        eta_min=0.08,
+        by_epoch=True,
+        begin=10,
+        end=max_search_epochs,
+        convert_to_iter_based=True),
+]
+
+# optimizer setting
+paramwise_cfg = dict(norm_decay_mult=0.0, bias_decay_mult=0.0)
+
+optim_wrapper = dict(
+    constructor='mmrazor.SeparateOptimWrapperConstructor',
+    architecture=dict(
+        type='OptimWrapper',
+        optimizer=dict(type='SGD', lr=0.5, momentum=0.9, weight_decay=3e-4),
+        paramwise_cfg=paramwise_cfg),
+    mutator=dict(
+        type='OptimWrapper',
+        optimizer=dict(type='Adam', lr=0.5, weight_decay=1e-3)))
+
+# data preprocessor
+data_preprocessor = dict(
+    type='mmcls.ClsDataPreprocessor',
+    # RGB format normalization parameters
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    # convert image from BGR to RGB
+    to_rgb=True,
+)
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='RandomResizedCrop', scale=224),
+    dict(type='ColorJitter', brightness=0.2, contrast=0.2, saturation=0.2),
+    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
+    dict(type='PackClsInputs'),
+]
+
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='ResizeEdge', scale=256, edge='short'),
+    dict(type='CenterCrop', crop_size=224),
+    dict(type='PackClsInputs'),
+]
+
+train_dataloader = dict(
+    batch_size=64,
+    num_workers=4,
+    dataset=dict(
+        type=dataset_type,
+        data_root='data/imagenet',
+        ann_file='meta/train.txt',
+        data_prefix='train',
+        pipeline=train_pipeline),
+    sampler=dict(type='DefaultSampler', shuffle=True, _scope_='mmcls'),
+    persistent_workers=True,
+)
+
+val_dataloader = dict(
+    batch_size=64,
+    num_workers=4,
+    dataset=dict(
+        type=dataset_type,
+        data_root='data/imagenet',
+        ann_file='meta/val.txt',
+        data_prefix='val',
+        pipeline=test_pipeline),
+    sampler=dict(type='DefaultSampler', shuffle=True, _scope_='mmcls'),
+    persistent_workers=True,
+)
+val_evaluator = dict(type='mmcls.Accuracy', topk=(1, 5))
+
+# If you want standard test, please manually configure the test dataset
+test_dataloader = val_dataloader
+test_evaluator = val_evaluator
+
+evaluation = dict(interval=1, metric='accuracy')
+
+train_cfg = dict(by_epoch=True, max_epochs=max_search_epochs, val_interval=1)
+val_cfg = dict()
+test_cfg = dict()
+custom_hooks = [dict(type='DMCPSubnetHook')]

configs/nas/mmcls/autoformer/AUTOFORMER_SUBNET_B.yaml

@@ -0,0 +1,134 @@
+backbone.base_embed_dims:
+  chosen: 64
+backbone.blocks.0.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.0.middle_channels:
+  chosen: 3.5
+backbone.blocks.0.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.0.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.1.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.1.middle_channels:
+  chosen: 3.5
+backbone.blocks.1.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.1.mutable_q_embed_dims:
+  chosen: 64
+backbone.blocks.10.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.10.middle_channels:
+  chosen: 4.0
+backbone.blocks.10.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.10.mutable_q_embed_dims:
+  chosen: 64
+backbone.blocks.11.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.11.middle_channels:
+  chosen: 576
+backbone.blocks.11.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.11.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.12.attn.mutable_attrs.num_heads:
+  chosen: 9
+backbone.blocks.12.middle_channels:
+  chosen: 4.0
+backbone.blocks.12.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.12.mutable_q_embed_dims:
+  chosen: 9
+backbone.blocks.13.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.13.middle_channels:
+  chosen: 4.0
+backbone.blocks.13.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.13.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.14.attn.mutable_attrs.num_heads:
+  chosen: 8
+backbone.blocks.14.middle_channels:
+  chosen: 576
+backbone.blocks.14.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.14.mutable_q_embed_dims:
+  chosen: 8
+backbone.blocks.15.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.15.middle_channels:
+  chosen: 3.0
+backbone.blocks.15.mutable_mlp_ratios:
+  chosen: 3.0
+backbone.blocks.15.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.2.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.2.middle_channels:
+  chosen: 576
+backbone.blocks.2.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.2.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.3.attn.mutable_attrs.num_heads:
+  chosen: 8
+backbone.blocks.3.middle_channels:
+  chosen: 4.0
+backbone.blocks.3.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.3.mutable_q_embed_dims:
+  chosen: 8
+backbone.blocks.4.attn.mutable_attrs.num_heads:
+  chosen: 10
+backbone.blocks.4.middle_channels:
+  chosen: 576
+backbone.blocks.4.mutable_mlp_ratios:
+  chosen: 3.0
+backbone.blocks.4.mutable_q_embed_dims:
+  chosen: 10
+backbone.blocks.5.attn.mutable_attrs.num_heads:
+  chosen: 9
+backbone.blocks.5.middle_channels:
+  chosen: 3.0
+backbone.blocks.5.mutable_mlp_ratios:
+  chosen: 3.0
+backbone.blocks.5.mutable_q_embed_dims:
+  chosen: 9
+backbone.blocks.6.attn.mutable_attrs.num_heads:
+  chosen: 8
+backbone.blocks.6.middle_channels:
+  chosen: 576
+backbone.blocks.6.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.6.mutable_q_embed_dims:
+  chosen: 8
+backbone.blocks.7.attn.mutable_attrs.num_heads:
+  chosen: 8
+backbone.blocks.7.middle_channels:
+  chosen: 3.5
+backbone.blocks.7.mutable_mlp_ratios:
+  chosen: 3.5
+backbone.blocks.7.mutable_q_embed_dims:
+  chosen: 8
+backbone.blocks.8.attn.mutable_attrs.num_heads:
+  chosen: 9
+backbone.blocks.8.middle_channels:
+  chosen: 576
+backbone.blocks.8.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.8.mutable_q_embed_dims:
+  chosen: 9
+backbone.blocks.9.attn.mutable_attrs.num_heads:
+  chosen: 8
+backbone.blocks.9.middle_channels:
+  chosen: 576
+backbone.blocks.9.mutable_mlp_ratios:
+  chosen: 4.0
+backbone.blocks.9.mutable_q_embed_dims:
+  chosen: 8
+backbone.mutable_depth:
+  chosen: 14
+backbone.mutable_embed_dims:
+  chosen: 576

configs/nas/mmcls/autoformer/README.md

@@ -44,15 +44,16 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 PORT=29500 ./tools/dist_train.sh \
 ```bash
 CUDA_VISIBLE_DEVICES=0 PORT=29500 ./tools/dist_test.sh \
   configs/nas/mmcls/autoformer/autoformer_subnet_8xb128_in1k.py \
-  $STEP2_CKPT 1 --work-dir $WORK_DIR \
-  --cfg-options algorithm.mutable_cfg=$STEP2_SUBNET_YAML
+  none 1 --work-dir $WORK_DIR \
+  --cfg-options model.init_cfg.checkpoint=$STEP1_CKPT model.init_weight_from_supernet=True
+
 ```
 
 ## Results and models
 
-| Dataset  | Supernet |                                                                                                                                               Subnet                                                                                                                                                | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) |                     Config                      | Download                                                                                                                                                                    |     Remarks      |
-| :------: | :------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :-------: | :------: | :-------: | :-------: | :---------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------: |
-| ImageNet |   vit    | [mutable](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/spos_shufflenetv2_subnet_8xb128_in1k/spos_shufflenetv2_subnet_8xb128_in1k_flops_0.33M_acc_73.87_20211222-454627be_mutable_cfg.yaml?versionId=CAEQHxiBgICw5b6I7xciIGY5MjVmNWFhY2U5MjQzN2M4NDViYzI2YWRmYWE1YzQx) |  52.472   |   10.2   |   82.48   |   95.99   | [config](./autoformer_supernet_32xb256_in1k.py) | [model](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/x.pth) \| [log](https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmrazor/v0.1/nas/spos/x.log.json) | MMRazor searched |
+| Dataset  | Supernet |                               Subnet                               | Params(M) | Flops(G) | Top-1 (%) | Top-5 (%) |                     Config                      | Download                                                                                                                                                                                                                                                  |     Remarks      |
+| :------: | :------: | :----------------------------------------------------------------: | :-------: | :------: | :-------: | :-------: | :---------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------: |
+| ImageNet |   vit    | [mutable](./configs/nas/mmcls/autoformer/AUTOFORMER_SUBNET_B.yaml) |  54.319   |  10.57   |   82.47   |   95.99   | [config](./autoformer_supernet_32xb256_in1k.py) | [model](https://download.openmmlab.com/mmrazor/v1/autoformer/autoformer_supernet_32xb256_in1k_20220919_110144-c658ce8f.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/autoformer/autoformer_supernet_32xb256_in1k_20220919_110144-c658ce8f.json) | MMRazor searched |
 
 **Note**:
 

configs/nas/mmcls/autoformer/autoformer_subnet_8xb256_in1k.py

@@ -0,0 +1,17 @@
+_base_ = 'autoformer_supernet_32xb256_in1k.py'
+
+model = dict(
+    _scope_='mmrazor',
+    type='sub_model',
+    cfg=_base_.supernet,
+    # NOTE: You can replace the yaml with the mutable_cfg searched by yourself
+    fix_subnet='configs/nas/mmcls/autoformer/AUTOFORMER_SUBNET_B.yaml',
+    # You can also load the checkpoint of supernet instead of the specific
+    # subnet by modifying the `checkpoint`(path) in the following `init_cfg`
+    # with `init_weight_from_supernet = True`.
+    init_weight_from_supernet=False,
+    init_cfg=dict(
+        type='Pretrained',
+        checkpoint=  # noqa: E251
+        'https://download.openmmlab.com/mmrazor/v1/autoformer/autoformer_supernet_32xb256_in1k_20220919_110144-c658ce8f.pth',  # noqa: E501
+        prefix='architecture.'))

configs/nas/mmcls/autoformer/autoformer_supernet_32xb256_in1k.py

@@ -52,7 +52,6 @@
 model = dict(
     type='mmrazor.Autoformer',
     architecture=supernet,
-    fix_subnet=None,
     mutator=dict(type='mmrazor.NasMutator'))
 
 # runtime setting

configs/nas/mmcls/autoslim/README.md

@@ -17,43 +17,43 @@ Notably, by setting optimized channel numbers, our AutoSlim-MobileNet-v2 at 305M
 
 ```bash
 CUDA_VISIBLE_DEVICES=0,1,2,3 PORT=29500 ./tools/dist_train.sh \
-  configs/pruning/autoslim/autoslim_mbv2_supernet_8xb256_in1k.py 4 \
+  configs/nas/autoslim/autoslim_mbv2_1.5x_supernet_8xb256_in1k.py 4 \
   --work-dir $WORK_DIR
 ```
 
 ### Search for subnet on the trained supernet
 
 ```bash
 CUDA_VISIBLE_DEVICES=0,1,2,3 PORT=29500 ./tools/dist_train.sh \
-  configs/pruning/autoslim/autoslim_mbv2_search_8xb1024_in1k.py 4 \
+  configs/nas/autoslim/autoslim_mbv2_1.5x_search_8xb256_in1k.py 4 \
   --work-dir $WORK_DIR --cfg-options load_from=$STEP1_CKPT
 ```
 
 ### Subnet retraining on ImageNet
 
 ```bash
 CUDA_VISIBLE_DEVICES=0,1,2,3 PORT=29500 ./tools/dist_train.sh \
-  configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py 4 \
+  configs/nas/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py 4 \
   --work-dir $WORK_DIR \
-  --cfg-options algorithm.channel_cfg=configs/pruning/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml,configs/pruning/autoslim/AUTOSLIM_MBV2_320M_OFFICIAL.yaml,configs/pruning/autoslim/AUTOSLIM_MBV2_220M_OFFICIAL.yaml
+  --cfg-options algorithm.channel_cfg=configs/nas/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml,configs/nas/autoslim/AUTOSLIM_MBV2_320M_OFFICIAL.yaml,configs/nas/autoslim/AUTOSLIM_MBV2_220M_OFFICIAL.yaml
 ```
 
 ### Split checkpoint
 
 ```bash
 python ./tools/model_converters/split_checkpoint.py \
-  configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py \
+  configs/nas/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py \
   $RETRAINED_CKPT \
-  --channel-cfgs configs/pruning/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml configs/pruning/autoslim/AUTOSLIM_MBV2_320M_OFFICIAL.yaml configs/pruning/autoslim/AUTOSLIM_MBV2_220M_OFFICIAL.yaml
+  --channel-cfgs configs/nas/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml configs/nas/autoslim/AUTOSLIM_MBV2_320M_OFFICIAL.yaml configs/nas/autoslim/AUTOSLIM_MBV2_220M_OFFICIAL.yaml
 ```
 
 ### Subnet inference
 
 ```bash
 CUDA_VISIBLE_DEVICES=0 PORT=29500 ./tools/dist_test.sh \
-  configs/pruning/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py \
+  configs/nas/autoslim/autoslim_mbv2_subnet_8xb256_in1k.py \
   $SEARCHED_CKPT 1 --work-dir $WORK_DIR \
-  --cfg-options algorithm.channel_cfg=configs/pruning/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml  # or modify the config directly
+  --cfg-options algorithm.channel_cfg=configs/nas/autoslim/AUTOSLIM_MBV2_530M_OFFICIAL.yaml  # or modify the config directly
 ```
 
 ## Results and models