Merge branch 'main' into clay-shields-sa-national

.github/workflows/build-preview.yml

@@ -0,0 +1,129 @@
+name: Build PR Preview
+on:
+  pull_request:
+    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+  workflow_dispatch:
+permissions:
+  contents: read
+  pages: write
+  pull-requests: write
+  id-token: write
+  checks: write
+concurrency: preview-${{ github.ref }}
+jobs:
+  build-preview:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+      - name: Checkout Main Branch 🛎️
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
+      - name: Checkout PR Branch 🛎️
+        uses: actions/checkout@v4
+        with:
+          path: pr-branch
+      - name: Install and Build Main Branch 🔧
+        run: |
+          npm ci --include=dev
+          npm run build
+          npm run style
+          npm run shields
+          cp src/configs/config.aws.js src/config.js
+        working-directory: main
+      - name: Install and Build PR Branch 🔧
+        # TODO: when we move shieldlib to its own repo, move shieldlib docs CI also
+        run: |
+          npm ci --include=dev
+          npm run build
+          npm run style
+          npm run shields
+          cp src/configs/config.aws.js src/config.js
+          mkdir -p dist/shield-docs
+          cp -r shieldlib/docs/* dist/shield-docs
+          mv dist ..
+        working-directory: pr-branch
+      - name: Capture main branch usage statistics
+        id: main-stats
+        run: |
+          MAIN_STATS=$(node ../pr-branch/scripts/stats.js -j)
+          echo "MAIN_STATS<<EOF" >> $GITHUB_ENV
+          echo -e "$MAIN_STATS" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        working-directory: main
+      - name: Start main branch map server
+        id: main-start-server
+        run: |
+          npm start &
+          until nc -z localhost 1776; do sleep 1; done
+        working-directory: main
+      - name: Capture main branch sample clips
+        # Run PR branch scripts against server running from main branch
+        id: main-samples
+        run: |
+          npm run generate_samples
+          mv samples ../samples-main
+        working-directory: pr-branch
+      - name: Stop main branch map server
+        id: main-stop-server
+        run: kill $(lsof -t -i:1776) 2>/dev/null || echo "No process found listening on port 1776."
+        working-directory: main
+      - name: Capture PR branch sample clips
+        id: pr-samples
+        run: |
+          npm start &
+          until nc -z localhost 1776; do sleep 1; done
+          npm run generate_samples
+          kill $(lsof -t -i:1776) 2>/dev/null || echo "No process found listening on port 1776."
+          mv samples ../samples-pr
+        working-directory: pr-branch
+      - name: Capture PR branch usage statistics
+        id: pr-stats
+        run: |
+          PR_STATS=$(node scripts/stats.js -j)
+          echo "PR_STATS<<EOF" >> $GITHUB_ENV
+          echo -e "$PR_STATS" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        working-directory: pr-branch
+      - name: Compare Stats
+        id: compare-stats
+        run: |
+          mkdir -p ../pr
+          echo '${{ env.MAIN_STATS }}'
+          echo '${{ env.PR_STATS }}'
+          npm exec tsx scripts/stats_compare '${{ env.MAIN_STATS }}' '${{ env.PR_STATS }}' > ../pr/stats-difference.md
+        working-directory: pr-branch
+      - name: Generate map diff sample clips
+        id: map-diff-samples
+        run: |
+          npm exec tsx scripts/folder_diff ../samples-main ../samples-pr https://preview.ourmap.us/pr/${{ github.event.pull_request.number }}/ ${{ github.event.pull_request.head.sha }}
+          mv pr_preview-extra.md ../pr/
+          cat ../pr/pr_preview-extra.md
+          mv samples-diff ../dist/
+        working-directory: pr-branch
+      - name: Upload Build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: americana
+          path: dist/
+      - name: Save PR attributes
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          echo $PR_NUMBER > pr/pr_number
+          echo $PR_SHA > pr/pr_sha
+          echo "Saved PR# ${{ github.event.pull_request.number }}, SHA# ${{ github.event.pull_request.head.sha }} for upload"
+      - name: Upload PR artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr_ci_artifacts
+          path: pr/

.github/workflows/deploy-pr-checks.yml

@@ -0,0 +1,143 @@
+name: Deploy PR Preview
+on:
+  workflow_run:
+    workflows: [Build PR Preview]
+    types:
+      - completed
+  workflow_dispatch:
+jobs:
+  deploy-preview:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    steps:
+      - name: "Download PR Number"
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "pr_ci_artifacts"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_ci_artifacts.zip`, Buffer.from(download.data));
+      - name: "Download Build"
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "americana"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/americana.zip`, Buffer.from(download.data));
+      - name: "Unzip artifacts"
+        run: |
+          unzip pr_ci_artifacts.zip
+          unzip -d dist americana.zip
+      - name: Set PR variable
+        run: |
+          if [ ! -f pr_number ] || [ ! -f pr_sha ]; then
+            echo "Required files pr_number or pr_sha do not exist. Exiting."
+            exit 1
+          fi
+          PR_NUM=$(cat pr_number)
+          PR_SHA=$(cat pr_sha)
+          if [ -z "$PR_NUM" ] || [ -z "$PR_SHA" ]; then
+            echo "PR_NUM or PR_SHA is empty. Exiting."
+            exit 1
+          fi
+          echo "Configuring deployment for PR# $PR_NUM commit $PR_SHA"
+          echo "PR_NUM=$PR_NUM" >> $GITHUB_ENV
+          echo "PR_SHA=$PR_SHA" >> $GITHUB_ENV
+      - name: Deploy via S3
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --follow-symlinks --delete --exclude '.git/*' --exclude '.github/*'
+        env:
+          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          SOURCE_DIR: "./dist"
+          DEST_DIR: pr/${{ env.PR_NUM }}
+      - name: Wait for PR Preview Upload (1x Sprite)
+        uses: cygnetdigital/wait_for_response@v2.0.0
+        with:
+          url: "https://preview.ourmap.us/pr/${{ env.PR_NUM }}/sprites/sprite.png"
+          responseCode: "200"
+          timeout: 120000
+          interval: 500
+      - name: Wait for PR Preview Upload (2x Sprite)
+        uses: cygnetdigital/wait_for_response@v2.0.0
+        with:
+          url: "https://preview.ourmap.us/pr/${{ env.PR_NUM }}/sprites/sprite@2x.png"
+          responseCode: "200"
+          timeout: 120000
+          interval: 500
+      - name: Generate Preview text
+        run: |
+          echo "## Live PR Preview:
+          * [Map](https://preview.ourmap.us/pr/${{ env.PR_NUM }}/)
+          * [Shield Test](https://preview.ourmap.us/pr/${{ env.PR_NUM }}/shieldtest.html)
+          * [style.json](https://preview.ourmap.us/pr/${{ env.PR_NUM }}/style.json)
+          * [shields.json](https://preview.ourmap.us/pr/${{ env.PR_NUM }}/shields.json)
+          * [taginfo.json](https://preview.ourmap.us/pr/${{ env.PR_NUM }}/taginfo.json)
+
+          Live previews are automatically removed once the PR is merged.
+
+          ## Sprite Sheets:
+
+          <img src="https://preview.ourmap.us/pr/${{ env.PR_NUM }}/sprites/sprite.png" />
+          <img src="https://preview.ourmap.us/pr/${{ env.PR_NUM }}/sprites/sprite@2x.png" />
+          " > pr_preview.md
+          [ -f pr_preview-extra.md ] && cat pr_preview-extra.md >> pr_preview.md || echo "No extra PR preview found."
+      - uses: tibdex/github-app-token@v1
+        id: checks_token
+        with:
+          app_id: 396440 #osm-americana checks app
+          private_key: ${{ secrets.CHECKS_WRITER_SECRET }}
+      - name: Print Preview Links to GitHub Checks
+        uses: LouisBrunner/checks-action@v1.6.1
+        if: always()
+        with:
+          sha: ${{ env.PR_SHA }}
+          token: ${{ steps.checks_token.outputs.token }}
+          name: PR Preview
+          details_url: https://preview.ourmap.us/pr/${{ env.PR_NUM }}/
+          conclusion: neutral
+          output: |
+            {"summary":"Preview map changes introduced by this PR", "title":"View live map and artifacts"}
+          output_text_description_file: pr_preview.md
+      - name: Print Stats to GitHub Checks
+        uses: LouisBrunner/checks-action@v1.6.1
+        if: always()
+        with:
+          sha: ${{ env.PR_SHA }}
+          token: ${{ steps.checks_token.outputs.token }}
+          name: Style Performance
+          conclusion: neutral
+          output: |
+            {"summary":"Style size changes introduced by this PR", "title":"View metrics on style size changes"}
+          output_text_description_file: stats-difference.md

.github/workflows/deploy.yml

@@ -9,29 +9,35 @@ permissions:
   contents: read
   pages: write
   id-token: write
+concurrency: preview-${{ github.ref }}
 jobs:
+  # TODO remove old PRs from s3 bucket automatically
   build-and-deploy:
     runs-on: ubuntu-latest
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
       - name: Checkout 🛎️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Use Node.js 18.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18.x
       - name: Install and Build 🔧
+        # TODO: when we move shieldlib to its own repo, move shieldlib docs CI also
         run: |
           npm ci --include=dev
           cp src/configs/config.aws.js src/config.js
           npm run build
           npm run style
+          npm run shields
+          mkdir -p dist/shield-docs
+          cp -r shieldlib/docs/* dist/shield-docs
       - name: Upload 🏗
-        uses: actions/upload-pages-artifact@v1
+        uses: actions/upload-pages-artifact@v3
         with:
           path: ./dist
       - name: Deploy 🚀
         id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v4

.github/workflows/lint.yml

@@ -10,20 +10,27 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           # Make sure the actual branch is checked out when running on pull requests
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
           # This is important to fetch the changes to the previous commit
           fetch-depth: 0
 
-      - name: Prettify code
-        uses: creyD/prettier_action@v3.3 # https://github.com/creyD/prettier_action
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
         with:
-          dry: true
-          prettier_options: --write .
-          # Setting only_changed to false picks up new files
-          only_changed: false
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          node-version: "18" # Specify the Node.js version
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run Prettier
+        id: prettier
+        run: npx prettier --check .
+        continue-on-error: false
+
+      - name: Check if Prettier failed
+        if: steps.prettier.outcome == 'failure'
+        run: exit 1

.github/workflows/s3-cleanup.yml

@@ -0,0 +1,24 @@
+name: Delete S3 Files on PR Merged
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-s3-files:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Remove files from S3
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          aws s3 rm s3://${{ secrets.AWS_S3_BUCKET }}/pr/${PR_NUMBER}/ --recursive