Automatically find and run special tests

azure-pipelines.yml

@@ -82,7 +82,7 @@ jobs:
       displayName: 'Testing: standard'
 
     - bash: |
-        sh tests/special_tests.sh
+        bash tests/special_tests.sh
       displayName: 'Testing: special'
 
     - bash: |

tests/special_tests.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Copyright The PyTorch Lightning team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,32 +12,51 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# Running special tests
 set -e
+
+# this environment variable allows special tests to run
 export PL_RUNNING_SPECIAL_TESTS=1
-DEFAULTS="-m coverage run --source pytorch_lightning --append -m pytest --verbose --capture=no"
-python ${DEFAULTS} tests/trainer/optimization/test_manual_optimization.py::test_step_with_optimizer_closure_with_different_frequencies_ddp
-python ${DEFAULTS} tests/models/test_sync_batchnorm.py::test_sync_batchnorm_ddp
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_invalid_deepspeed_defaults_no_precision
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_warn_deepspeed_override_backward
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_deepspeed_run_configure_optimizers
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_deepspeed_config
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_deepspeed_custom_precision_params
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_deepspeed_assert_config_zero_offload_disabled
-python ${DEFAULTS} tests/plugins/test_deepspeed_plugin.py::test_deepspeed_multigpu
-python ${DEFAULTS} tests/plugins/test_rpc_plugin.py::test_rpc_function_calls_ddp
-python ${DEFAULTS} tests/plugins/test_rpc_sequential_plugin.py::test_rpc_sequential_plugin_manual
-python ${DEFAULTS} tests/plugins/test_rpc_sequential_plugin.py::test_rpc_sequential_plugin_manual_amp
-python ${DEFAULTS} tests/plugins/test_rpc_sequential_plugin.py::test_rpc_sequential_plugin_automatic
-python ${DEFAULTS} tests/plugins/test_rpc_sequential_plugin.py::test_rpc_sequential_plugin_with_wrong_balance
-python ${DEFAULTS} tests/utilities/test_all_gather_grad.py::test_all_gather_collection
-python ${DEFAULTS} tests/trainer/test_trainer.py::test_trainer_predict_ddp
-python ${DEFAULTS} tests/trainer/test_trainer.py::test_trainer_predict_dp
-python ${DEFAULTS} tests/trainer/logging_/test_train_loop_logging_1_0.py::test_logging_sync_dist_true_ddp
-python ${DEFAULTS} tests/callbacks/test_pruning.py::test_pruning_callback_ddp
-python ${DEFAULTS} tests/test_profiler.py::test_pytorch_profiler_trainer_ddp
-python ${DEFAULTS} tests/models/test_hooks.py::test_transfer_batch_hook_ddp
-python ${DEFAULTS} tests/trainer/test_data_loading.py::test_replace_distrubuted_sampler_custom_dataloader_custom_batch_sampler
-python ${DEFAULTS} tests/trainer/optimization/test_manual_optimization.py::test_step_with_optimizer_closure_with_different_frequencies_ddp_with_toggle_model
-python ${DEFAULTS} tests/checkpointing/test_checkpoint_callback_frequency.py::test_top_k_ddp
-nvprof --profile-from-start off -o trace_name.prof -- python ${DEFAULTS} tests/test_profiler.py::test_pytorch_profiler_nested_emit_nvtx
+# python arguments
+defaults='-m coverage run --source pytorch_lightning --append -m pytest --verbose --capture=no'
+
+# find tests marked as `@RunIf(special=True)`
+grep_output=$(grep --recursive --line-number --word-regexp 'tests' --regexp 'special=True' | grep '@RunIf')
+# file paths
+files=$(echo "$grep_output" | cut -f1 -d:)
+files_arr=($files)
+# line numbers
+linenos=$(echo "$grep_output" | cut -f2 -d:)
+linenos_arr=($linenos)
+
+# tests to skip - space separated
+blocklist='test_pytorch_profiler_nested_emit_nvtx'
+
+for i in "${!files_arr[@]}"; do
+  file=${files_arr[$i]}
+  lineno=${linenos_arr[$i]}
+
+  echo "Found $file:$lineno"
+
+  # get code from `@RunIf(special=True)` line to EOF
+  test_code=$(tail -n +"$lineno" "$file")
+
+  # read line by line
+  while read -r line; do
+    # if it's a test
+    if [[ $line == def\ test_* ]]; then
+      # get the name
+      test_name=$(echo $line | cut -c 5- | cut -f1 -d\()
+
+      # check blocklist
+      if echo $blocklist | grep --word-regexp "$test_name" > /dev/null; then
+        break
+      fi
+
+      # run the test
+      python ${defaults} "${file}::${test_name}"
+      break
+    fi
+  done < <(echo "$test_code")
+done
+
+nvprof --profile-from-start off -o trace_name.prof -- python ${defaults} tests/test_profiler.py::test_pytorch_profiler_nested_emit_nvtx

tests/utilities/test_all_gather_grad.py

@@ -115,6 +115,6 @@ def training_step(self, batch, batch_idx):
             return loss
 
     model = TestModel()
-    trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, gpus=2)
+    trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True, gpus=2, accelerator="ddp")
     trainer.fit(model)
     assert model.training_step_called