Update tests (#3217)

* add missing daily gpu test

* log

* logs

* remove failed

* add mlflow fixture

* fix tboard test

* fix test

* fix test

* run test

* rename more tests

* add patch

* upgrade

* more upgrade

.github/workflows/code-quality.yaml

@@ -34,7 +34,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.4
+        ref: v0.0.5
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/code-quality
       with:

.github/workflows/codeql-analysis.yml

@@ -45,7 +45,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.4
+        ref: v0.0.5
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/codeql-analysis
       with:

.github/workflows/coverage.yaml

@@ -16,7 +16,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.4
+        ref: v0.0.5
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/coverage
       with:

.github/workflows/daily.yaml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   daily-pytest-cpu:
-    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.4
+    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.5
     strategy:
       matrix:
         include:
@@ -100,7 +100,7 @@ jobs:
       download-path: artifacts
 
   daily-pytest-gpu:
-    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.4
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.5
     strategy:
       matrix:
         # Unlike CPU tests, we run daily tests together with GPU tests to minimize launch time
@@ -112,7 +112,12 @@ jobs:
           pytest_command: "coverage run -m pytest"
           composer_package_name: "mosaicml"
         - name: "gpu-3.11-2.2"
-          container: mosaicml/pytorch:2.2.0_cu121-python3.11-ubuntu20.04
+          container: mosaicml/pytorch:2.2.1_cu121-python3.11-ubuntu20.04
+          markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
+          pytest_command: "coverage run -m pytest"
+          composer_package_name: "mosaicml"
+        - name: "gpu-3.11-2.3"
+          container: mosaicml/pytorch:2.3.0_cu121-python3.11-ubuntu20.04
           markers: "(daily or not daily) and (remote or not remote) and gpu and (doctest or not doctest)"
           pytest_command: "coverage run -m pytest"
           composer_package_name: "mosaicml"

.github/workflows/pr-cpu.yaml

@@ -9,7 +9,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   pytest-cpu:
-    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.4
+    uses: mosaicml/ci-testing/.github/workflows/pytest-cpu.yaml@v0.0.5
     strategy:
       matrix:
         include:

.github/workflows/pr-gpu.yaml

@@ -9,7 +9,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   pytest-gpu:
-    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.4
+    uses: mosaicml/ci-testing/.github/workflows/pytest-gpu.yaml@v0.0.5
     strategy:
       matrix:
         include:
@@ -24,7 +24,7 @@ jobs:
       composer_package_name: ${{ matrix.composer_package_name }}
       container: ${{ matrix.container }}
       git_repo: mosaicml/composer
-      mcloud-timeout: 1500
+      mcloud-timeout: 2250
       name: ${{ matrix.name }}
       pip_deps: "[all]"
       pytest-command: ${{ matrix.pytest_command }}

.github/workflows/release.yaml

@@ -24,7 +24,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.4
+        ref: v0.0.5
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/code-quality
       with:

.github/workflows/smoketest.yaml

@@ -33,7 +33,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.4
+        ref: v0.0.5
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/smoketest
       with:

tests/fixtures/autouse_fixtures.py

@@ -46,6 +46,17 @@ def disable_wandb(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureReques
             monkeypatch.setenv('WANDB_PROJECT', 'pytest')
 
 
+@pytest.fixture(autouse=True)
+def reset_mlflow_tracking_dir():
+    """Reset MLFlow tracking dir so it doesn't persist across tests"""
+    try:
+        import mlflow
+        mlflow.set_tracking_uri(None)  # type: ignore
+    except ModuleNotFoundError:
+        # MLFlow not installed
+        pass
+
+
 @pytest.fixture(scope='session')
 def cleanup_dist():
     """Ensure all dist tests clean up resources properly."""

tests/loggers/test_mlflow_logger.py

@@ -43,7 +43,7 @@ def _get_latest_mlflow_run(experiment_name, tracking_uri=None):
         raise ValueError(f'Experiment with name {experiment_name} is unexpectedly empty')
 
 
-def test_mlflow_experiment_init_unspecified(monkeypatch):
+def test_mlflow_init_unspecified(monkeypatch):
     """ Test that MLFlow experiment is set up correctly when no parameters are specified
 
     This mocks the mlflow library to check that the correct calls are made to set up the experiment
@@ -73,7 +73,7 @@ def test_mlflow_experiment_init_unspecified(monkeypatch):
     )
 
 
-def test_mlflow_experiment_init_specified(monkeypatch):
+def test_mlflow_init_specified(monkeypatch):
     """ Test that MLFlow experiment is set up correctly when all parameters are specified
 
     This mocks the mlflow library to check that the correct calls are made to set up the experiment
@@ -114,7 +114,7 @@ def test_mlflow_experiment_init_specified(monkeypatch):
     )
 
 
-def test_mlflow_experiment_init_ids(monkeypatch):
+def test_mlflow_init_ids(monkeypatch):
     """ Test that MLFlow experiment is set up correctly when ids in the environment are specified
 
     This mocks the mlflow library to check that the correct calls are made to set up the experiment
@@ -144,7 +144,7 @@ def test_mlflow_experiment_init_ids(monkeypatch):
     assert mlflow.start_run.called_with(run_id=mlflow_run_id)
 
 
-def test_mlflow_experiment_init_experiment_name(monkeypatch):
+def test_mlflow_init_experiment_name(monkeypatch):
     """ Test that MLFlow experiment is set up correctly when experiment name is specified
 
     This mocks the mlflow library to check that the correct calls are made to set up the experiment
@@ -170,7 +170,7 @@ def test_mlflow_experiment_init_experiment_name(monkeypatch):
     id_logger.post_close()
 
 
-def test_mlflow_experiment_init_existing_composer_run(monkeypatch):
+def test_mlflow_init_existing_composer_run(monkeypatch):
     """ Test that an existing MLFlow run is used if one tagged with `run_name` exists in the experiment for the Composer run.
     """
     mlflow = pytest.importorskip('mlflow')
@@ -238,7 +238,7 @@ def test_mlflow_logger_uses_state_run_name_if_no_env_var_set(monkeypatch, mock_m
     assert test_logger.tags['run_name'] == 'state-run-name'
 
 
-def test_mlflow_experiment_set_up(tmp_path):
+def test_mlflow_set_up(tmp_path):
     """ Test that MLFlow experiment is set up correctly within mlflow
     """
     mlflow = pytest.importorskip('mlflow')

tests/loggers/test_tensorboard_logger.py

@@ -19,7 +19,7 @@ def test_tensorboard_logger(tmp_path, dummy_state):
     return tensorboard_logger
 
 
-def test_tensorboard_log_image(test_tensorboard_logger):
+def test_tensorboard_log_image(test_tensorboard_logger, dummy_state):
     pytest.importorskip('tensorboard', reason='tensorboard is optional')
 
     image_variants = [
@@ -49,5 +49,6 @@ def test_tensorboard_log_image(test_tensorboard_logger):
             channels_last=channels_last,
         )
 
-    test_tensorboard_logger.post_close()
+    logger = Logger(dummy_state, [])
+    test_tensorboard_logger.close(dummy_state, logger)
     # Tensorboard images are stored inline, so we can't check them automatically.