Merge pull request #4658 from ESMCI/fix_workflow_python

Fixes creating new environment with specific python version

.github/workflows/testing.yml

@@ -113,10 +113,11 @@ jobs:
           export CIME_REMOTE=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}
           export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}}
 
-          mamba install -y python=${{ matrix.python-version }}
-
           source /entrypoint.sh
 
+          # from 'entrypoint.sh', create and activate new environment
+          create_environment ${{ matrix.python-version }}
+
           # GitHub runner home is different than container
           cp -rf /root/.cime /github/home/
 

docker/Dockerfile

@@ -1,5 +1,5 @@
-ARG MAMBAFORGE_VERSION=4.14.0-0
-FROM condaforge/mambaforge:${MAMBAFORGE_VERSION} AS base
+ARG BASE_TAG=24.3.0-0
+FROM condaforge/miniforge3:${BASE_TAG} AS base
 
 SHELL ["/bin/bash", "-c"]
 
@@ -12,52 +12,15 @@ RUN mkdir -p \
     wget -O /storage/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc && \
     wget -O /storage/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc
 
-# Install common packages
-RUN mamba install --yes -c conda-forge \
-            cmake \
-            make \
-            wget \
-            curl \
-            subversion \
-            m4 \
-            pytest \
-            pytest-cov\
-            pyyaml \
-            vim \
-            rsync \
-            openssh && \
-            rm -rf /opt/conda/pkgs/*
-
-# Compilers and libraries
-ARG LIBNETCDF_VERSION=4.9.1
-ENV LIBNETCDF_VERSION=${LIBNETCDF_VERSION}
-ARG NETCDF_FORTRAN_VERSION=*
-ENV NETCDF_FORTRAN_VERSION=${NETCDF_FORTRAN_VERSION}
-ARG ESMF_VERSION=*
-ENV ESMF_VERSION=${ESMF_VERSION}
-ARG GCC_VERSION=10.*
-ENV GCC_VERSION=${GCC_VERSION}
-
-# Install version locked packages
-# gcc, gxx, gfortran provide symlinks for x86_64-conda-linux-gnu-*
-# ar and ranlib are not symlinked
-RUN mamba install --yes -c conda-forge \
-            lapack \
-            blas \
-            libnetcdf=${LIBNETCDF_VERSION}=*openmpi* \
-            netcdf-fortran=${NETCDF_FORTRAN_VERSION}=*openmpi* \
-            esmf=${ESMF_VERSION}=*openmpi* \
-            gcc_linux-64=${GCC_VERSION} \
-            gxx_linux-64=${GCC_VERSION} \
-            openmpi-mpifort \
-            gfortran_linux-64=${GCC_VERSION} \
-            gcc \
-            gxx \
-            gfortran && \
-            rm -rf /opt/conda/pkgs/* && \
-            ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ar /opt/conda/bin/ar && \
-            ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ranlib /opt/conda/bin/ranlib && \
-            cpan install XML::LibXML Switch
+
+COPY cime.yaml /cime.yaml
+
+RUN mamba env update -f /cime.yaml && \
+    rm -rf /opt/conda/pkgs/* && \
+    ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ar /opt/conda/bin/ar && \
+    ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ranlib /opt/conda/bin/ranlib && \
+    # need compilers
+    cpan install XML::LibXML Switch
 
 ARG PNETCDF_VERSION=1.12.3
 ENV PNETCDF_VERSION=${PNETCDF_VERSION}
@@ -99,7 +62,7 @@ COPY entrypoint.sh /entrypoint.sh
 
 ENTRYPOINT [ "/entrypoint.sh" ]
 
-FROM base as slurm
+FROM base AS slurm
 
 RUN apt-get update && \
             DEBIAN_FRONTEND=noninteractive \
@@ -112,7 +75,7 @@ COPY slurm/slurm.conf /etc/slurm-llnl/
 COPY slurm/config_batch.xml /root/.cime/
 COPY slurm/entrypoint_batch.sh /entrypoint_batch.sh
 
-FROM base as pbs
+FROM base AS pbs
 
 RUN apt-get update && \
             DEBIAN_FRONTEND=noninteractive \

docker/cime.yaml

@@ -0,0 +1,29 @@
+name: base
+channels:
+  - conda-forge
+dependencies:
+  - cmake
+  - make
+  - wget
+  - curl
+  - subversion
+  - m4
+  - pytest
+  - pytest-cov
+  - pyyaml
+  - vim
+  - rsync
+  - openssh
+  - lapack
+  - blas
+  - libnetcdf=4.9.1=*openmpi*
+  - netcdf-fortran=*=*openmpi*
+  - esmf=*=*openmpi*
+  - gcc_linux-64=10.*
+  - gxx_linux-64=10.*
+  - gfortran_linux-64=10.*
+  - openmpi-mpifort
+  - gcc
+  - gxx
+  - gfortran
+prefix: /opt/conda

docker/entrypoint.sh

@@ -134,4 +134,13 @@ then
     . "/entrypoint_batch.sh"
 fi
 
+function create_environment() {
+    mamba create -n cime-$1 python=$1
+    mamba env update -n cime-$1 -f /cime.yaml
+
+    source /opt/conda/etc/profile.d/conda.sh
+
+    conda activate cime-$1
+}
+
 exec "${@}"