Container

.all-contributorsrc.json

@@ -0,0 +1,23 @@
+{
+  "projectName": "all-contributors",
+  "projectOwner": "all-contributors",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": ["CONTRIBUTORS.md"],
+  "imageSize": 100,
+  "commit": false,
+  "contributorsPerLine": 7,
+  "contributorsSortAlphabetically": false,
+  "badgeTemplate": "[![All Contributors](https://img.shields.io/badge/all_contributors-<%= contributors.length %>-orange.svg?style=flat-square)](#contributors)",
+  "contributorTemplate": "<a href=\"<%= contributor.profile %>\"><img src=\"<%= contributor.avatar_url %>\" width=\"<%= options.imageSize %>px;\" alt=\"\"/><br /><sub><b><%= contributor.name %></b></sub></a>",
+  "types": {
+    "custom": {
+      "symbol": "🔭",
+      "description": "A custom contribution type.",
+      "link": "[<%= symbol %>](<%= url %> \"<%= description %>\"),"
+    }
+  },
+  "linkToUsage": true,
+  "skipCi": true,
+  "contributors": []
+}

.github/workflows/docker-build-push-aws.yml

@@ -0,0 +1,53 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Build and push image to Amazon ECR.
+
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  AWS_REGION: us-east-1                  # set this to your preferred AWS region, e.g. us-west-1
+  ECR_REPOSITORY: mdtf/mdtf-test        # set this to your Amazon ECR repository name
+  ECS_SERVICE: custom-service                # MY_ECS_SERVICE   set this to your Amazon ECS service name
+  ECS_CLUSTER: mdtf-east1                 # MY_ECS_CLUSTER  set this to your Amazon ECS cluster name
+  ECS_TASK_DEFINITION: ecs-task-defn.json # set this to the path to your Amazon ECS task definition
+                                               # file, e.g. .aws/task-definition.json
+  CONTAINER_NAME: mdtf-test-a1r           # set this to the name of the container in the
+                                               # containerDefinitions section of your task definition MY_CONTAINER_NAME
+
+jobs:
+  deploy:
+    name: push-to-public-ecr-cache-test
+    runs-on: ubuntu-latest
+    environment: dev
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@13d241b293754004c80624b5567555c4a39ffbe3
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        uses: aradhakrishnanGFDL/docker-build-with-cache-action@master #support-aws-ecr-public
+        env:
+         ECR_REGISTRY: public.ecr.aws
+         IMAGE_TAG: "latest"
+         ECS_IMAGE: $ECR_REGISTRY/ECR_REPOSITORY:$env.IMAGE_TAG
+        with:
+         registry: public.ecr.aws
+         username: "${{ secrets.AWS_ACCESS_KEY_ID }}"  
+         password: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"  
+         image_name: c6h4h4s8/mdtf/mdtf-test
+         push_image_and_stages: true
+         image_tag: latest

.github/workflows/docker-run-test.yml

@@ -0,0 +1,18 @@
+name: docker-run-minitest1
+on:
+  pull_request:
+    branches: [ main ]
+jobs:
+  container-test-job:
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/c6h4h4s8/mdtf/mdtf-test:latest
+      env:
+        NODE_ENV: development
+      options: --cpus 1
+    steps:
+      - name: Check PATH
+        run: (echo $PATH) || (echo "No path")
+      - name: Activate environment
+        shell: bash 
+        run: (eval "$(micromamba shell hook -s bash -p ~/conda)"; micromamba activate; micromamba activate _MDTF_base;mdtf_framework.py --version)

.github/workflows/mdtf_test_cloud.yml

@@ -0,0 +1,175 @@
+# This file builds and runs a lightweight version of the MDTF test suite.
+# Note that the tests assess functionality of the diagnostics,
+# and do not evaluate data or scientific content.
+name: MDTF_test_cloud
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+    branches:
+      - main
+      - develop
+defaults:
+  run:
+    shell: bash -l {0}
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ami-0d9c66960c53029ef
+          ec2-instance-type: t2.xlarge
+          subnet-id: subnet-00d803f7c597997c8 #######subnet-003faaf9f1e386979
+          security-group-id: sg-016e543200ed56a0f
+          #iam-role-name: github-actions-role-a # optional, requires additional permissions
+
+  build:
+    name: MDTF core tests
+    needs: start-runner
+    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+  #  continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]#, /usr/share/miniconda3, /Users/runner/miniconda3]
+        action-conda-root: [/actions-runner/_work/MDTF-diagnostics/MDTF-diagnostics/3]
+        json-file: ["tests/github_actions_test_cloud_set1.jsonc","tests/github_actions_test_macos_set1.jsonc"]
+        json-file-set2: ["tests/github_actions_test_ubuntu_set2.jsonc", "tests/github_actions_test_macos_set2.jsonc"]
+        json-file-set3: ["tests/github_actions_test_ubuntu_set3.jsonc", "tests/github_actions_test_macos_set3.jsonc"]
+        # if experimental is true, other jobs to run if one fails
+        experimental: [false]
+        exclude:
+          - os: ubuntu-latest
+            conda-root: /Users/runner/miniconda3
+          - os: ubuntu-latest
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - os: ubuntu-latest
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - os: ubuntu-latest
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file: "tests/github_actions_test_macos_set1.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file: "tests/github_actions_test_ubuntu_set1.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set2: "tests/github_actions_test_macos_set2.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set2: "tests/github_actions_test_ubuntu_set2.jsonc"
+          - conda-root: /usr/share/miniconda3
+            json-file-set3: "tests/github_actions_test_macos_set3.jsonc"
+          - conda-root: /Users/runner/minconda3
+            json-file-set3: "tests/github_actions_test_ubuntu_set3.jsonc"
+      max-parallel: 2
+    steps:
+    - uses: actions/checkout@v2
+    - name: Download Miniconda 3
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniconda-version: "latest"
+        python-version: 3.8
+    - name: Verify miniconda
+      run: |
+        conda info -a
+        conda config --show
+    - name: Set environment variables
+      run: |
+        echo "POD_OUTPUT=$(echo $PWD/../wkdir/GFDL.Synthetic)" >> $GITHUB_ENV
+    - name: Install Conda Environments
+      run: |
+        # install mamba (https://github.com/mamba-org/mamba) for faster dependency resolution
+        echo "Installing Mamba"
+        conda install mamba -n base -c conda-forge
+        # create the synthetic data environment
+        echo "Creating the _MDTF_synthetic_data environment"
+        mamba env create --force -q -f ./src/conda/_env_synthetic_data.yml
+        echo "Installing Conda Environments"
+        # MDTF-specific setup: install all conda envs
+        ##${{matrix.conda-root}}
+        ./src/conda/conda_env_setup.sh --all --conda_root ${{matrix.action-conda-root}} 
+    - name: Generate Model Data
+      run: |
+        cd ../
+        conda activate _MDTF_synthetic_data
+        pip install mdtf-test-data
+        mkdir mdtf_test_data ; cd mdtf_test_data
+        # generate the data and run unit tests
+        mdtf_synthetic.py -c GFDL --startyear 1 --nyears 10 --unittest
+        mdtf_synthetic.py -c NCAR --startyear 1975 --nyears 7
+        mdtf_synthetic.py -c CMIP --startyear 1 --nyears 10
+        cd ../
+        mkdir wkdir
+    - name: Get Observational Data for Set 1
+      run: |
+        echo "${PWD}"
+        cd ../
+        echo "Available Space"
+        df -h
+        # attempt FTP data fetch
+        # allow 20 min for transfer before timeout; Github actions allows 6 hours for individual
+        # jobs, but we don't want to max out resources that are shared by the NOAA-GFDL repos.
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/convective_transition_diag_obs_data.tar --output convective_transition_diag_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/EOF_500hPa_obs_data.tar --output EOF_500hPa_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/Wheeler_Kiladis_obs_data.tar --output Wheeler_Kiladis_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_teleconnection_obs_data.tar --output MJO_teleconnection_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/MJO_suite_obs_data.tar --output MJO_suite_obs_data.tar
+        curl --verbose --ipv4 --connect-timeout 8 --max-time 1200 --retry 128 --ftp-ssl --disable-epsv --ftp-pasv -u "anonymous:anonymous" ftp://ftp.gfdl.noaa.gov/perm/oar.gfdl.mdtf/precip_diurnal_cycle_obs_data.tar --output precip_diurnal_cycle_obs_data.tar
+        ## make input data directories
+        mkdir -p inputdata/obs_data
+        echo "Untarring test files"
+        tar -xvf convective_transition_diag_obs_data.tar
+        tar -xvf EOF_500hPa_obs_data.tar
+        tar -xvf precip_diurnal_cycle_obs_data.tar
+        tar -xvf MJO_teleconnection_obs_data.tar
+        tar -xvf MJO_suite_obs_data.tar
+        tar -xvf Wheeler_Kiladis_obs_data.tar
+        # clean up tarballs
+        rm -f *.tar
+    - name: Run diagnostic tests set 1
+      run: |
+        echo "POD_OUTPUT is: "
+        echo "${POD_OUTPUT}"
+        conda activate _MDTF_base
+        cat ./mdtf
+        # trivial check that install script worked
+        ./mdtf --version
+        # run the test PODs
+        ./mdtf -v -f ${{matrix.json-file}}
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - build # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}     

CONTRIBUTORS.md

@@ -0,0 +1,10 @@
+## Contributors
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->

Dockerfile

@@ -0,0 +1,48 @@
+# Base OS
+FROM ubuntu:latest
+FROM mambaorg/micromamba:latest as micromamba
+
+#micromamba set up
+
+USER root
+
+RUN micromamba info
+
+
+RUN micromamba shell hook --shell bash
+#RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml
+
+##
+# Container Metadata
+LABEL maintainer="20195932+wrongkindofdoctor@users.noreply.github.com"
+LABEL version="alpha-01"
+LABEL description="This is a docker image for the MDTF-diagnostics package"
+# Disable Prompt During Packages Installation
+#ARG DEBIAN_FRONTEND=noninteractive
+
+# Copy the MDTF-diagnostics package contents from local machine to image
+ENV CODE_ROOT=/proj/MDTF-diagnostics
+COPY src ${CODE_ROOT}/src                          
+COPY data ${CODE_ROOT}/data
+#COPY diagnostics ${CODE_ROOT}/diagnostics
+COPY mdtf_framework.py ${CODE_ROOT}
+COPY shared ${CODE_ROOT}/shared
+COPY sites ${CODE_ROOT}/sites
+COPY tests ${CODE_ROOT}/tests
+# Install conda environments
+ENV CONDA_ROOT=/opt/conda/
+ENV CONDA_ENV_DIR=/opt/conda/envs
+
+#USER mambauser
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_base.yml
+RUN micromamba create -f /proj/MDTF-diagnostics/src/conda/env_NCL_base.yml
+
+
+ENV PATH="${PATH}:/proj/MDTF-diagnostics/"
+#cRUN micromamba activate _MDTF_base
+# Verify installation
+#RUN /proj/MDTF-diagnostics/mdtf_framework.py --help
+# Run mdtf on src/default_tests.jsonc
+# CMD ["${CODE_ROOT}/mdtf", "-f","${CODE_ROOT}/src/default_tests.jsonc"]
+ENTRYPOINT ["micromamba activate _MDTF_base"]
+CMD ["/bin/bash"]