From 5b539bc5294023fd3a996a9ddb3a7b21fceb68e8 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 11 Oct 2024 17:46:28 -0600 Subject: [PATCH 1/7] Update chicoma-cpu modules --- cime_config/machines/config_machines.xml | 50 ++++++++++++++---------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7af65ddcccbf..1d754646473e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3957,11 +3957,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -3973,26 +3973,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 + PrgEnv-gnu/8.5.0 gcc/12.2.0 cray-libsci/23.05.1.4 @@ -4004,8 +4005,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4014,15 +4015,22 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + + craype-accel-host + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + + + craype-accel-host craype/2.7.21 cray-mpich/8.1.26 - libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 @@ -4044,6 +4052,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + -1 @@ -4085,11 +4096,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4156,7 +4167,6 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 From a1d98a723a4d74a1f6e5c56e97c2241475354ec9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 14 Oct 2024 14:43:58 -0600 Subject: [PATCH 2/7] Add latest cmake --- cime_config/machines/config_machines.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 1d754646473e..4764a7028762 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4022,6 +4022,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4031,6 +4032,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 @@ -4167,6 +4169,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + cmake/3.27.7 From 9c0b30854137acb603412e3ef213794b7613c0cd Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Thu, 17 Oct 2024 09:34:47 -0600 Subject: [PATCH 3/7] Updates to get E3SM working on chicoma-cpu --- cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake | 1 + cime_config/machines/config_machines.xml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb76..a6c13942620f 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 4764a7028762..234d3ba09f14 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3937,7 +3937,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -4022,7 +4022,6 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - cmake/3.27.7 @@ -4032,6 +4031,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 + + + cmake/3.27.7 From c5552b9c781ae5a397dc8e42121819308373bdd2 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 19 Oct 2024 15:07:53 -0600 Subject: [PATCH 4/7] Update nvidia modules --- cime_config/machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 234d3ba09f14..9bfaa798ac0c 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3999,8 +3999,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - PrgEnv-nvidia/8.4.0 - nvidia/22.7 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 cray-libsci/23.05.1.4 From ca879585b4dd6d6501b77a89407715ea51ea1958 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 18:31:01 -0600 Subject: [PATCH 5/7] Add GNU_CRAY_LDFLAGS flag to MPAS standalone builds This should be set to: ``` export GNU_CRAY_LDFLAGS="-Wl,--enable-new-dtags" ``` on Chicoma-CPU with gcc. --- components/mpas-framework/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/mpas-framework/Makefile b/components/mpas-framework/Makefile index 1f9b00844e72..f92606987bbb 100644 --- a/components/mpas-framework/Makefile +++ b/components/mpas-framework/Makefile @@ -396,11 +396,11 @@ gnu-cray: "FFLAGS_OPT = -O3 -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_OPT = -O3 -m64" \ "CXXFLAGS_OPT = -O3 -m64" \ - "LDFLAGS_OPT = -O3 -m64" \ + "LDFLAGS_OPT = -O3 -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_DEBUG = -g -m64 -ffree-line-length-none -fconvert=big-endian -ffree-form -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow -ffpe-summary=none $${EXTRA_FFLAGS}" \ "CFLAGS_DEBUG = -g -m64" \ "CXXFLAGS_DEBUG = -g -m64" \ - "LDFLAGS_DEBUG = -g -m64" \ + "LDFLAGS_DEBUG = -g -m64 $(GNU_CRAY_LDFLAGS)" \ "FFLAGS_OMP = -fopenmp" \ "CFLAGS_OMP = -fopenmp" \ "BUILD_TARGET = $(@)" \ From d379003d79649c6c76f0993a0d1134f04486ac26 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 20 Oct 2024 21:43:12 -0600 Subject: [PATCH 6/7] Revert chicoma-gpu changes (for now) --- cime_config/machines/config_machines.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 9bfaa798ac0c..41ee2b618e3e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4100,11 +4100,11 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' - /usr/share/lmod/lmod/init/perl + /usr/share/lmod/8.3.1/init/perl - /usr/share/lmod/lmod/init/python - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh + /usr/share/lmod/8.3.1/init/python + /usr/share/lmod/8.3.1/init/sh + /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -4171,7 +4171,7 @@ AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100' cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 - cmake/3.27.7 + cmake/3.25.1 From c34336e47dea80b58c8b039bc1a76fd78b82bf1d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 21 Oct 2024 11:44:41 -0600 Subject: [PATCH 7/7] Update gcc and nvidia to newer versions This is to match proposed updates to Perlmutter-CPU https://github.com/E3SM-Project/E3SM/pull/6702 --- cime_config/machines/config_machines.xml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 41ee2b618e3e..a16a5fe0b67a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -3973,6 +3973,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 + gcc + gcc-native intel intel-oneapi nvidia @@ -3994,14 +3996,14 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + gcc-native/12.3 + cray-libsci/23.12.5 PrgEnv-nvidia/8.5.0 nvidia/24.7 - cray-libsci/23.05.1.4 + cray-libsci/23.12.5 @@ -4015,25 +4017,13 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-libsci/23.05.1.4 - + craype-accel-host craype/2.7.30 cray-mpich/8.1.28 cray-hdf5-parallel/1.12.2.9 cray-netcdf-hdf5parallel/4.9.0.9 cray-parallel-netcdf/1.12.3.9 - - - - craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - - - cmake/3.27.7 @@ -4057,7 +4047,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_PARALLEL_NETCDF_PREFIX} - /opt/cray/pe/gcc/12.2.0/snos/lib64:$ENV{LD_LIBRARY_PATH} + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} -1