test: report individual test results from python tests (#9366)

to ci
MD-343

.circleci/real_config.yml

@@ -650,8 +650,11 @@ commands:
           command: |
             pytest -vv -s \
             -m <<parameters.mark>> \
+            --junitxml=/tmp/test-results/det-deploy-tests.xml \
             --no-compare-stats \
             --det-version="<<parameters.det-version>>"
+      - store_test_results:
+          path: /tmp/test-results/
 
   deploy-aws-cluster:
     parameters:
@@ -2311,6 +2314,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-cpu-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-gpu-tf:
     docker:
@@ -2333,6 +2338,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-gpu-tf-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-pytorch2-gpu:
     docker:
@@ -2355,6 +2362,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-pytorch2-gpu-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-pytorch2-cpu:
     docker:
@@ -2376,6 +2385,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-pytorch2-cpu-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-gpu-parallel:
     docker:
@@ -2398,6 +2409,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-gpu-parallel-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-gpu-deepspeed:
     docker:
@@ -2420,6 +2433,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-gpu-deepspeed-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-harness-tf2:
     docker:
@@ -2441,6 +2456,8 @@ jobs:
           root: .
           paths:
             - test-unit-harness-tf2-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-storage:
     docker:
@@ -2457,13 +2474,15 @@ jobs:
           determined: true
           extra-requirements-file: "harness/tests/requirements/requirements-harness.txt"
           executor: <<pipeline.parameters.docker-image>>
-      - run: COVERAGE_FILE=$PWD/test-unit-storage-pycov coverage run -m pytest -v --durations=0 --require-secrets -m cloud harness/tests
+      - run: COVERAGE_FILE=$PWD/test-unit-storage-pycov coverage run -m pytest -v --durations=0 --junitxml=/tmp/test-results/test-unit-storage.xml --require-secrets -m cloud harness/tests
       - run: coverage xml -i --data-file=./test-unit-storage-pycov
       - run: codecov -v -t $CODECOV_TOKEN -F harness
       - persist_to_workspace:
           root: .
           paths:
             - test-unit-storage-pycov
+      - store_test_results:
+          path: /tmp/test-results
 
   test-unit-model-hub:
     docker:
@@ -2539,7 +2558,9 @@ jobs:
       - run: pip install -r harness/tests/requirements/requirements-cli.txt
       - run: pip freeze --all
       - python-report
-      - run: pytest harness/tests/cli
+      - run: pytest --junitxml=/tmp/test-results/test-unit-storage.xml harness/tests/cli
+      - store_test_results:
+          path: /tmp/test-results
 
   test-e2e-slurm-disabled:
     parameters:

harness/Makefile

@@ -1,3 +1,4 @@
+TEST_RESULTS_DIR=/tmp/test-results
 py_bindings_dest=determined/common/api/bindings.py
 cuda_available=$(shell python -c "import torch; print(torch.cuda.is_available())") \
 
@@ -29,37 +30,46 @@ check: check-gen
 	python -m flake8
 	mypy .
 
+
 .PHONY: test-cpu
 test-cpu:
-	coverage run -m pytest -v --runslow --durations=0 -m "not gpu or cpu" tests
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m "not gpu or cpu" --junitxml=$(TEST_RESULTS_DIR)/test-cpu.xml tests
 
 .PHONY: test-gpu-tf
 test-gpu-tf:
-	coverage run -m pytest -v --runslow --durations=0 -m "gpu" tests/experiment/keras
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m "gpu" --junitxml=$(TEST_RESULTS_DIR)/test-gpu.xml tests/experiment/keras
 
 .PHONY: test-gpu-deepspeed
 test-gpu-deepspeed:
-	coverage run -m pytest -v --runslow --durations=0 -m "deepspeed and gpu" tests/experiment/integrations/test_deepspeed_trial.py
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m "deepspeed and gpu" --junitxml=$(TEST_RESULTS_DIR)/test-gpu-deepspeed.xml tests/experiment/integrations/test_deepspeed_trial.py
 
 .PHONY: test-gpu-parallel
 test-gpu-parallel:
-	coverage run -m pytest -v --runslow --durations=0 -m "gpu_parallel" tests/experiment/pytorch
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m tests/experiment -m "gpu_parallel" --junitxml=$(TEST_RESULTS_DIR)/test-gpu-parallel.xml tests/experiment/pytorch
 
 .PHONY: test-pytorch-cpu
 test-pytorch-cpu:
-	coverage run -m pytest -v --runslow --durations=0 -m tests/experiment -m "not gpu or cpu" tests/experiment/pytorch
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m tests/experiment -m "not gpu or cpu" --junitxml=$(TEST_RESULTS_DIR)/test-pytorch-cpu.xml tests/experiment/pytorch
 
 .PHONY: test-pytorch-gpu
 test-pytorch-gpu:
-	coverage run -m pytest -v --runslow --durations=0 -m tests/experiment -m "gpu" tests/experiment/pytorch
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 -m tests/experiment -m "gpu" --junitxml=$(TEST_RESULTS_DIR)/test-pytorch-gpu.xml tests/experiment/pytorch
 
 .PHONY: test
 test:
-	coverage run -m pytest -v --runslow --durations=0 tests
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -m pytest -v --runslow --durations=0 --junitxml=$(TEST_RESULTS_DIR)/test.xml tests
 
 .PHONY: test-tf2
 test-tf2:
-	coverage run -a -m pytest -v --runslow --durations=0 -m "not gpu" tests/experiment/keras
+	mkdir -p $(TEST_RESULTS_DIR)
+	coverage run -a -m pytest -v --runslow --durations=0 -m "not gpu" --junitxml=$(TEST_RESULTS_DIR)/test-tf2.xml tests/experiment/keras
 
 .PHONY: clean
 clean: