chore: bumpenv update NGC base images version to 24.03 (#9132)

determined-ai · Apr 17, 2024 · 1037d83 · 1037d83
1 parent 1cc9cd7
commit 1037d83
Show file tree

Hide file tree

Showing 34 changed files with 217 additions and 209 deletions.
diff --git a/.circleci/real_config.yml b/.circleci/real_config.yml
@@ -37,7 +37,7 @@ parameters:
   # be referenced by --ee testing.
   default-pt-gpu-image:
     type: string
-    default: determinedai/environments:cuda-11.3-pytorch-1.12-gpu-079eb6d
+    default: determinedai/environments:cuda-11.3-pytorch-1.12-gpu-748dda4
   # Some python, go, and react dependencies are cached by circleci via `save_cache`/`restore_cache`.
   # If the dependencies stay the same, but the circleci code that would produce them is changed,
   # it may be necessary to invalidate the cache by incrementing this value.
@@ -201,7 +201,7 @@ commands:
       - when:
           condition: <<parameters.tf2>>
           steps:
-            - run: docker pull determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-079eb6d
+            - run: docker pull determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-748dda4
 
   login-docker:
     parameters:
@@ -1988,7 +1988,7 @@ jobs:
 
   test-unit-harness-gpu:
     docker:
-      - image: determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1
+      - image: determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1
     resource_class: determined-ai/container-runner-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -2010,7 +2010,7 @@ jobs:
 
   test-unit-harness-pytorch2-gpu:
     docker:
-      - image: determinedai/environments:cuda-11.8-pytorch-2.0-gpu-079eb6d
+      - image: determinedai/environments:cuda-11.8-pytorch-2.0-gpu-748dda4
     resource_class: determined-ai/container-runner-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
@@ -2032,7 +2032,7 @@ jobs:
 
   test-unit-harness-pytorch2-cpu:
     docker:
-      - image: determinedai/environments:py-3.10-pytorch-2.0-cpu-079eb6d
+      - image: determinedai/environments:py-3.10-pytorch-2.0-cpu-748dda4
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
       - checkout
@@ -2053,7 +2053,7 @@ jobs:
 
   test-unit-harness-gpu-parallel:
     docker:
-      - image: determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1
+      - image: determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1
     resource_class: determined-ai/container-runner-multi-gpu
     steps:
       - run: mkdir -p ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts

diff --git a/docs/model-dev-guide/api-guides/apis-howto/_index.rst b/docs/model-dev-guide/api-guides/apis-howto/_index.rst
@@ -76,13 +76,13 @@ experiment is controlled by the container image that has been configured for tha
 Determined provides prebuilt Docker images that include TensorFlow 2.11, 1.15, and 2.8,
 respectively:
 
--  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1`` (default)
+-  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1`` (default)
 -  ``determinedai/environments:cuda-10.2-pytorch-1.7-tf-1.15-gpu-0.21.2``
 -  ``determinedai/environments:cuda-11.2-tf-2.8-gpu-0.29.1``
 
 We also provide lightweight CPU-only counterparts:
 
--  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1``
+-  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1``
 -  ``determinedai/environments:py-3.8-tf-2.8-cpu-0.29.1``
 
 To change the container image used for an experiment, specify :ref:`environment.image

diff --git a/docs/model-dev-guide/prepare-container/custom-env.rst b/docs/model-dev-guide/prepare-container/custom-env.rst
@@ -101,9 +101,9 @@ Default Images
 +-------------+-------------------------------------------------------------------------------+
 | Environment | File Name                                                                     |
 +=============+===============================================================================+
-| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1``          |
+| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1``          |
 +-------------+-------------------------------------------------------------------------------+
-| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1``       |
+| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1``       |
 +-------------+-------------------------------------------------------------------------------+
 | AMD GPUs    | ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4``        |
 +-------------+-------------------------------------------------------------------------------+
@@ -132,7 +132,7 @@ Example Dockerfile that installs custom ``conda``-, ``pip``-, and ``apt``-based
 .. code:: bash
 
    # Determined Image
-   FROM determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1
+   FROM determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1
 
    # Custom Configuration
    RUN apt-get update && \
@@ -195,7 +195,7 @@ environments using :ref:`custom images <custom-docker-images>`:
 .. code:: bash
 
    # Determined Image
-   FROM determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1
+   FROM determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1
 
    # Create a virtual environment
    RUN conda create -n myenv python=3.8

diff --git a/docs/reference/deploy/helm-config-reference.rst b/docs/reference/deploy/helm-config-reference.rst
@@ -194,13 +194,13 @@
 
    -  ``cpuImage``: Sets the default Docker image for all non-GPU tasks. If a Docker image is
       specified in the :ref:`experiment config <exp-environment-image>` this default is overriden.
-      Defaults to: ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1``.
+      Defaults to: ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1``.
 
    -  ``startupHook``: An optional inline script that will be executed as part of task set up.
 
    -  ``gpuImage``: Sets the default Docker image for all GPU tasks. If a Docker image is specified
       in the :ref:`experiment config <exp-environment-image>` this default is overriden. Defaults
-      to: ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1``.
+      to: ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1``.
 
    -  ``logPolicies``: Sets log policies for trials. For details, visit :ref:`log_policies
       <experiment-config-min-validation-period>`.

diff --git a/docs/reference/deploy/master-config-reference.rst b/docs/reference/deploy/master-config-reference.rst
@@ -89,9 +89,9 @@ configure different container images for NVIDIA GPU tasks using the ``cuda`` key
 Determined 0.17.6), CPU tasks using ``cpu`` key, and ROCm (AMD GPU) tasks using the ``rocm`` key.
 Default values:
 
--  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1`` for NVIDIA GPUs.
+-  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1`` for NVIDIA GPUs.
 -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
--  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1`` for CPUs.
+-  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1`` for CPUs.
 
 ``environment_variables``
 =========================

diff --git a/docs/reference/experiment-config-reference.rst b/docs/reference/experiment-config-reference.rst
@@ -1333,8 +1333,8 @@ Optional. The Docker image to use when executing the workload. This image must b
 container images for NVIDIA GPU tasks using ``cuda`` key (``gpu`` prior to 0.17.6), CPU tasks using
 ``cpu`` key, and ROCm (AMD GPU) tasks using ``rocm`` key. Default values:
 
--  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1`` for NVIDIA GPUs.
--  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1`` for CPUs.
+-  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1`` for NVIDIA GPUs.
+-  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1`` for CPUs.
 -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
 
 When the cluster is configured with :ref:`resource_manager.type: slurm

diff --git a/docs/reference/job-config-reference.rst b/docs/reference/job-config-reference.rst
@@ -45,9 +45,9 @@ The following configuration settings are supported:
       different container images for NVIDIA GPU tasks using ``cuda`` key (``gpu`` prior to 0.17.6),
       CPU tasks using ``cpu`` key, and ROCm (AMD GPU) tasks using ``rocm`` key. Default values:
 
-      -  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1`` for NVIDIA GPUs.
+      -  ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1`` for NVIDIA GPUs.
       -  ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-0.26.4`` for ROCm.
-      -  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1`` for CPUs.
+      -  ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1`` for CPUs.
 
    -  ``force_pull_image``: Forcibly pull the image from the Docker registry and bypass the Docker
       cache. Defaults to ``false``.

diff --git a/docs/setup-cluster/deploy-cluster/slurm/singularity.rst b/docs/setup-cluster/deploy-cluster/slurm/singularity.rst
@@ -26,9 +26,9 @@ by default in this version of Determined are described below.
 +-------------+--------------------------------------------------------------------------+
 | Environment | File Name                                                                |
 +=============+==========================================================================+
-| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-079eb6d``    |
+| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-748dda4``    |
 +-------------+--------------------------------------------------------------------------+
-| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-079eb6d`` |
+| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-748dda4`` |
 +-------------+--------------------------------------------------------------------------+
 | AMD GPUs    | ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512``  |
 +-------------+--------------------------------------------------------------------------+

diff --git a/docs/setup-cluster/gcp/install-gcp.rst b/docs/setup-cluster/gcp/install-gcp.rst
@@ -406,5 +406,5 @@ This command line will spin up a cluster of up to 2 A100s in the ``us-central1-c
       --compute-agent-instance-type a2-highgpu-1g --gpu-num 1 \
       --gpu-type nvidia-tesla-a100 \
       --region us-central1 --zone us-central1-c \
-      --gpu-env-image determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.30.1 \
-      --cpu-env-image determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.30.1
+      --gpu-env-image determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-0.31.1 \
+      --cpu-env-image determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-0.31.1
diff --git a/docs/setup-cluster/slurm/singularity.rst b/docs/setup-cluster/slurm/singularity.rst
@@ -26,9 +26,9 @@ by default in this version of Determined are described below.
 +-------------+--------------------------------------------------------------------------+
 | Environment | File Name                                                                |
 +=============+==========================================================================+
-| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-079eb6d``    |
+| CPUs        | ``determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-748dda4``    |
 +-------------+--------------------------------------------------------------------------+
-| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-079eb6d`` |
+| NVIDIA GPUs | ``determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-748dda4`` |
 +-------------+--------------------------------------------------------------------------+
 | AMD GPUs    | ``determinedai/environments:rocm-5.0-pytorch-1.10-tf-2.7-rocm-622d512``  |
 +-------------+--------------------------------------------------------------------------+

diff --git a/docs/setup-cluster/slurm/slurm-requirements.rst b/docs/setup-cluster/slurm/slurm-requirements.rst
@@ -510,7 +510,7 @@ platform. There may be additional per-user configuration that is required.
 
    .. code:: bash
 
-      image=determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-079eb6d
+      image=determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-748dda4
       cd /shared/enroot/images
       enroot import docker://$image
       enroot create /shared/enroot/images/${image//[\/:]/\+}.sqsh

diff --git a/e2e_tests/tests/config.py b/e2e_tests/tests/config.py
@@ -14,12 +14,12 @@
 MAX_TRIAL_BUILD_SECS = 90
 
 
-DEFAULT_TF2_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-079eb6d"
-DEFAULT_TF2_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-079eb6d"
-DEFAULT_PT_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-cpu-079eb6d"
-DEFAULT_PT_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-gpu-079eb6d"
-DEFAULT_PT2_CPU_IMAGE = "determinedai/environments:py-3.10-pytorch-2.0-cpu-079eb6d"
-DEFAULT_PT2_GPU_IMAGE = "determinedai/environments:cuda-11.8-pytorch-2.0-gpu-079eb6d"
+DEFAULT_TF2_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-tf-2.11-cpu-748dda4"
+DEFAULT_TF2_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-tf-2.11-gpu-748dda4"
+DEFAULT_PT_CPU_IMAGE = "determinedai/environments:py-3.9-pytorch-1.12-cpu-748dda4"
+DEFAULT_PT_GPU_IMAGE = "determinedai/environments:cuda-11.3-pytorch-1.12-gpu-748dda4"
+DEFAULT_PT2_CPU_IMAGE = "determinedai/environments:py-3.10-pytorch-2.0-cpu-748dda4"
+DEFAULT_PT2_GPU_IMAGE = "determinedai/environments:cuda-11.8-pytorch-2.0-gpu-748dda4"
 
 TF2_CPU_IMAGE = os.environ.get("TF2_CPU_IMAGE") or DEFAULT_TF2_CPU_IMAGE
 TF2_GPU_IMAGE = os.environ.get("TF2_GPU_IMAGE") or DEFAULT_TF2_GPU_IMAGE

diff --git a/harness/determined/deploy/aws/templates/efs.yaml b/harness/determined/deploy/aws/templates/efs.yaml
@@ -3,35 +3,35 @@ Mappings:
   RegionMap:
     ap-northeast-1:
       Master: ami-00910ef9457f0df47
-      Agent: ami-0aae1314bf4cd46a8
+      Agent: ami-0d16e1e0dcb55d0cc
     # TODO(DET-4258) Uncomment these when we fully support all P3 regions.
     # ap-northeast-2:
     #   Master: ami-035e3e44dc41db6a2
-    #   Agent: ami-01a7dc54f4832d921
+    #   Agent: ami-0b5e1d346243cc56d
     # ap-southeast-1:
     #   Master: ami-0fd1ee6c8b656f020
-    #   Agent: ami-08c7512a3a706763d
+    #   Agent: ami-0c82409e1680a5f05
     # ap-southeast-2:
     #   Master: ami-0b62ecd3babd1c548
-    #   Agent: ami-08d850301481cfa6f
+    #   Agent: ami-0fb880629a676b994
     eu-central-1:
       Master: ami-0abbe417ed83c0b29
-      Agent: ami-0c3642bfef1e23d05
+      Agent: ami-06b9510e1e2ff459a
     eu-west-1:
       Master: ami-0e3f7dd2dc743e48a
-      Agent: ami-0f9e5e94c7d911407
+      Agent: ami-0cdb3c6edfed8684a
     # eu-west-2:
     #   Master: ami-0d78429fb6af30994
-    #   Agent: ami-001ac62ef6c960d70
+    #   Agent: ami-0b7adfc10fc8f8cae
     us-east-1:
       Master: ami-0172070f66a8ebe63
-      Agent: ami-00681263a44d2b064
+      Agent: ami-080fa6043c3db5762
     us-east-2:
       Master: ami-0bafa3699418551cd
-      Agent: ami-03b081710dc515cb7
+      Agent: ami-03fedff28640be3f8
     us-west-2:
       Master: ami-0ceeab680f529cc36
-      Agent: ami-0f0978d073ce78ae4
+      Agent: ami-03c80bd6abfc63658
 
 Parameters:
   VpcCIDR:

diff --git a/harness/determined/deploy/aws/templates/fsx.yaml b/harness/determined/deploy/aws/templates/fsx.yaml
@@ -3,35 +3,35 @@ Mappings:
   RegionMap:
     ap-northeast-1:
       Master: ami-00910ef9457f0df47
-      Agent: ami-0aae1314bf4cd46a8
+      Agent: ami-0d16e1e0dcb55d0cc
     # TODO(DET-4258) Uncomment these when we fully support all P3 regions.
     # ap-northeast-2:
     #   Master: ami-035e3e44dc41db6a2
-    #   Agent: ami-01a7dc54f4832d921
+    #   Agent: ami-0b5e1d346243cc56d
     # ap-southeast-1:
     #   Master: ami-0fd1ee6c8b656f020
-    #   Agent: ami-08c7512a3a706763d
+    #   Agent: ami-0c82409e1680a5f05
     # ap-southeast-2:
     #   Master: ami-0b62ecd3babd1c548
-    #   Agent: ami-08d850301481cfa6f
+    #   Agent: ami-0fb880629a676b994
     eu-central-1:
       Master: ami-0abbe417ed83c0b29
-      Agent: ami-0c3642bfef1e23d05
+      Agent: ami-06b9510e1e2ff459a
     eu-west-1:
       Master: ami-0e3f7dd2dc743e48a
-      Agent: ami-0f9e5e94c7d911407
+      Agent: ami-0cdb3c6edfed8684a
     # eu-west-2:
     #   Master: ami-0d78429fb6af30994
-    #   Agent: ami-001ac62ef6c960d70
+    #   Agent: ami-0b7adfc10fc8f8cae
     us-east-1:
       Master: ami-0172070f66a8ebe63
-      Agent: ami-00681263a44d2b064
+      Agent: ami-080fa6043c3db5762
     us-east-2:
       Master: ami-0bafa3699418551cd
-      Agent: ami-03b081710dc515cb7
+      Agent: ami-03fedff28640be3f8
     us-west-2:
       Master: ami-0ceeab680f529cc36
-      Agent: ami-0f0978d073ce78ae4
+      Agent: ami-03c80bd6abfc63658
 
 Parameters:
   VpcCIDR:

diff --git a/harness/determined/deploy/aws/templates/govcloud.yaml b/harness/determined/deploy/aws/templates/govcloud.yaml
@@ -5,10 +5,10 @@ Mappings:
   RegionMap:
     us-gov-east-1:
       Master: ami-04ef693ebcf519dc3
-      Agent: ami-047875a86e970378f
+      Agent: ami-0db37b18a422d2157
     us-gov-west-1:
       Master: ami-08bd15d820a3c087e
-      Agent: ami-0aa306aba45ce531c
+      Agent: ami-05736ebda877e4663
 Parameters:
   Keypair:
     Type: AWS::EC2::KeyPair::KeyName

diff --git a/harness/determined/deploy/aws/templates/lore.yaml b/harness/determined/deploy/aws/templates/lore.yaml
@@ -3,35 +3,35 @@ Mappings:
   RegionMap:
     ap-northeast-1:
       Master: ami-00910ef9457f0df47
-      Agent: ami-0aae1314bf4cd46a8
+      Agent: ami-0d16e1e0dcb55d0cc
     # TODO(DET-4258) Uncomment these when we fully support all P3 regions.
     # ap-northeast-2:
     #   Master: ami-035e3e44dc41db6a2
-    #   Agent: ami-01a7dc54f4832d921
+    #   Agent: ami-0b5e1d346243cc56d
     # ap-southeast-1:
     #   Master: ami-0fd1ee6c8b656f020
-    #   Agent: ami-08c7512a3a706763d
+    #   Agent: ami-0c82409e1680a5f05
     # ap-southeast-2:
     #   Master: ami-0b62ecd3babd1c548
-    #   Agent: ami-08d850301481cfa6f
+    #   Agent: ami-0fb880629a676b994
     eu-central-1:
       Master: ami-0abbe417ed83c0b29
-      Agent: ami-0c3642bfef1e23d05
+      Agent: ami-06b9510e1e2ff459a
     eu-west-1:
       Master: ami-0e3f7dd2dc743e48a
-      Agent: ami-0f9e5e94c7d911407
+      Agent: ami-0cdb3c6edfed8684a
     # eu-west-2:
     #   Master: ami-0d78429fb6af30994
-    #   Agent: ami-001ac62ef6c960d70
+    #   Agent: ami-0b7adfc10fc8f8cae
     us-east-1:
       Master: ami-0172070f66a8ebe63
-      Agent: ami-00681263a44d2b064
+      Agent: ami-080fa6043c3db5762
     us-east-2:
       Master: ami-0bafa3699418551cd
-      Agent: ami-03b081710dc515cb7
+      Agent: ami-03fedff28640be3f8
     us-west-2:
       Master: ami-0ceeab680f529cc36
-      Agent: ami-0f0978d073ce78ae4
+      Agent: ami-03c80bd6abfc63658
 
 Parameters:
   VpcCIDR: