Skip to content

test some ideas

test some ideas #17684

Workflow file for this run

name: Go Workflows
# note, because of generate tests using path filters here is too complicated
on:
pull_request:
push:
branches-ignore:
- 'gh-pages'
jobs:
cancel-outdated:
name: Cancel Outdated Jobs
runs-on: ubuntu-latest
steps:
- id: skip_check
if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && !contains(github.event.head_commit.message, '[no_skip]') }}
uses: fkirc/skip-duplicate-actions@v5
with:
cancel_others: 'true'
# changes allows us to only run a job on changed packages
changes:
name: Change Detection
runs-on: ubuntu-latest
outputs:
# Expose matched filters as job 'packages' output variable
packages_deps: ${{ steps.filter_go_deps.outputs.changed_modules }}
packages_nodeps: ${{ steps.filter_go_nodeps.outputs.changed_modules }}
package_count_deps: ${{ steps.length.outputs.FILTER_LENGTH_DEPS }}
package_count_nodeps: ${{ steps.length.outputs.FILTER_LENGTH_NODEPS }}
solidity_changes: ${{ steps.filter_solidity.outputs.any_changed }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: 'recursive'
- uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest
id: filter_go_deps
with:
include_deps: true
github_token: ${{ secrets.WORKFLOW_PAT }}
timeout: '10m'
- uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest
id: filter_go_nodeps
with:
include_deps: false
github_token: ${{ secrets.WORKFLOW_PAT }}
timeout: '10m'
- name: Check For Solidity Changes
id: filter_solidity
uses: tj-actions/changed-files@v26.1
with:
files: |
**/*.sol
- name: Run step if any of the listed files above change
if: steps.filter_solidity.outputs.any_changed == 'true'
run: |
echo "One or more files listed above has changed."
- id: length
run: |
export FILTER_LENGTH_DEPS=$(echo $FILTERED_PATHS_DEPS | jq '. | length')
echo "##[set-output name=FILTER_LENGTH_DEPS;]$(echo $FILTER_LENGTH_DEPS)"
export FILTER_LENGTH_NODEPS=$(echo $FILTERED_PATHS_NODEPS | jq '. | length')
echo "##[set-output name=FILTER_LENGTH_NODEPS;]$(echo $FILTER_LENGTH_NODEPS)"
env:
FILTERED_PATHS_DEPS: ${{ steps.filter_go_deps.outputs.changed_modules }}
FILTERED_PATHS_NODEPS: ${{ steps.filter_go_nodeps.outputs.changed_modules }}
test:
name: Go Coverage
runs-on: ${{ matrix.platform }}
if: ${{ needs.changes.outputs.package_count_deps > 0 }}
needs: changes
strategy:
fail-fast: false
matrix:
go-version:
- 1.21.x
platform:
- ubuntu-latest
# only do on agents for now. Anything that relies on solidity in a package should do this
package: ${{ fromJSON(needs.changes.outputs.packages_deps) }}
services:
mariadb:
image: mariadb:10.11.3
ports:
- 3306
env:
MYSQL_USER: user
MYSQL_PASSWORD: password
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
options: --health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=30
defaults:
run:
working-directory: ${{ matrix.package }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
submodules: 'recursive'
- name: Cache Docker images.
uses: ScribeMD/docker-cache@0.2.6
with:
key: docker-test-${{ runner.os }}-${{ matrix.package }}
- name: Go modules cache
uses: actions/cache@v3
with:
# see https://github.com/mvdan/github-actions-golang
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-test-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-test-${{matrix.package}}
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Verify MariaDB connection
env:
PORT: ${{ job.services.mariadb.ports[3306] }}
run: |
while ! mysqladmin ping -h"127.0.0.1" -P"$PORT" --silent; do
sleep 1
done
- name: Install pyroscope-ci
uses: jaxxstorm/action-install-gh-release@v1.5.0
with: # Grab the latest version
repo: pyroscope-io/ci
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Get Coverage name
id: coverage
run: |
echo "::set-output name=flag::$(echo "${{ matrix.package }}" | rev | cut -d/ -f1 | rev)"
- name: Get pyroscope path
id: pyroscope-path
run: echo "##[set-output name=PYROSCOPE_PATH;]$(which pyroscope-ci)"
shell: bash
- name: Setup Pyroscope Tests
run: $PYROSCOPE_PATH go install --applicationName=$APPLICATION_NAME .
shell: bash
env:
PYROSCOPE_PATH: '${{ steps.pyroscope-path.outputs.PYROSCOPE_PATH }}'
APPLICATION_NAME: ${{ steps.coverage.outputs.flag }}
- name: Test
uses: nick-fields/retry@v2
with:
command: cd ${{matrix.package}} && $PYROSCOPE_PATH exec --apiKey=${{ secrets.PYROSCOPE_CLOUD_TOKEN }} -- go test -parallel 1 -coverpkg=./... ./... -coverprofile=profile.cov
max_attempts: 2
timeout_minutes: 20
env:
PYROSCOPE_PATH: '${{ steps.pyroscope-path.outputs.PYROSCOPE_PATH }}'
ENABLE_MYSQL_TEST: true
MYSQL_HOST: 0.0.0.0
MYSQL_USER: user
MYSQL_PASSWORD: password
MYSQL_DATABASE: test
MYSQL_ROOT_PASSWORD: password
MYSQL_PORT: ${{ job.services.mariadb.ports[3306] }}
GOMEMLIMIT: 3GiB
GOGC: -1
ETHEREUM_RPC_URI: ${{ secrets.ETHEREUM_RPC_URI }}
GOPROXY: https://proxy.golang.org
- name: Send Coverage (Codecov)
uses: Wandalen/wretry.action@v1.0.36
with:
action: codecov/codecov-action@v3
with: |
token: ${{ secrets.CODECOV }}
files: profile.cov
flags: ${{ steps.coverage.outputs.flag }}
fail_ci_if_error: true # optional (default = false)
verbose: true # optional (default = false)
attempt_limit: 5
attempt_delay: 30000
# make sure the build works
build:
name: Build
needs: changes
runs-on: ${{ matrix.platform }}
if: ${{ needs.changes.outputs.package_count_deps > 0 }}
strategy:
fail-fast: false
matrix:
go-version:
- 1.21.x
platform:
- ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Go modules cache
uses: actions/cache@v3
with:
# see https://github.com/mvdan/github-actions-golang
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-build-
- name: Build
# go build all workspaces
run: go build $(go work edit -json | jq -c -r '[.Use[].DiskPath] | map_values(. + "/...")[]')
# see: https://www.reddit.com/r/golang/comments/476pae/how_to_speed_up_go_compiler_and_many_other_go/
# this cannot use gomemlimit because we are running multiple builds in paralell
env:
GOGC: 2000
GOMEMLIMIT: 6GiB
#note: right now this is not run against all work dirs
lint:
name: Lint
runs-on: ubuntu-latest
needs: changes
if: ${{ needs.changes.outputs.package_count_nodeps > 0 }}
strategy:
fail-fast: false
matrix:
# Parse JSON array containing names of all filters matching any of changed files
# e.g. ['package1', 'package2'] if both package folders contains changes
package: ${{ fromJSON(needs.changes.outputs.packages_nodeps) }}
steps:
- uses: actions/setup-go@v3
with:
# see: https://github.com/golangci/golangci-lint/issues/3420, moving to go 1.20 requires a new golangci-lint version
# TODO: with this being 3 behind this should be done sooner rather than later.
go-version: 1.19.x
- uses: actions/checkout@v3
with:
fetch-depth: 0
# workaround for: https://github.com/golangci/golangci-lint-action/issues/479
- name: Setup cache key
run: cp ${{matrix.package}}/go.mod go.mod -v
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
working-directory: ${{matrix.package}}/
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: v1.48.0
# see: https://github.com/golangci/golangci-lint/issues/2654
args: --timeout=60m
env:
# GitHub token for annotations (optional)
GITHUB_TOKEN: ${{ secrets.WORKFLOW_PAT }}
GOMEMLIMIT: 6GiB
GOGC: -1
issue_number:
# this is needed to prevent us from hitting the github api rate limit
name: Get The Issue Number
needs: changes
runs-on: ubuntu-latest
# currently, this matches the logic in the go generate check. If we ever add more checks that run on all packages, we should
# change this to run on those pushes
if: ${{ github.event_name != 'pull_request' && (needs.changes.outputs.solidity_changes == 'true' || needs.changes.outputs.package_count_deps > 0 ) }}
outputs:
issue_number: ${{ steps.find_pr.outputs.pr }}
steps:
- uses: jwalton/gh-find-current-pr@v1
id: find_pr
# check if we need to rerun go generate as a result of solidity changes. Note, this will only run on solidity changes.
# TODO: consolidate w/ go change check. This will run twice on agents
check-generation-solidity:
name: Go Generate (Solidity Only)
runs-on: ubuntu-latest
needs: [changes, issue_number]
if: ${{ github.event_name != 'pull_request' && needs.changes.outputs.solidity_changes == 'true' }}
strategy:
fail-fast: false
matrix:
# only do on agents for now. Anything that relies on solidity in a package should do this
package: ['agents']
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
submodules: 'recursive'
- name: Cache Docker images.
uses: ScribeMD/docker-cache@0.2.6
with:
key: docker-generate-${{ runner.os }}-${{ matrix.package }}
- name: Setup NodeJS
uses: ./.github/actions/setup-nodejs
# Foundry is required for flattening
- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1
with:
version: nightly
# Generate flattened files
- name: Run flattener
run: npx lerna exec npm run build:go --parallel
# Setup Go
- uses: actions/setup-go@v3
with:
go-version: 1.21.x
- name: Go modules cache
uses: actions/cache@v3
with:
# see https://github.com/mvdan/github-actions-golang
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
# use seperate cache for generate, builds less stuff
# TODO: consider scoping to package
key: ${{ runner.os }}-go-generate-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-generate-${{matrix.package}}
# TODO: remove
- name: authenticate with github for private go modules
if: ${{github.event.repository.private}}
uses: fusion-engineering/setup-git-credentials@v2
with:
credentials: https://trajan0x:${{secrets.GIT_TOKEN }}@github.com/
# See if we need to rerun go generate
# TODO: consider implementing https://github.com/golang/go/issues/20520 to speed up process if possible
- name: Try Go Generate
working-directory: ${{matrix.package}}/
run: |
go generate ./...
env:
GOMEMLIMIT: 6GiB
GOGC: -1
- name: Verify Changed files
uses: tj-actions/verify-changed-files@v11.1
id: verify-changed-files
with:
files: |
*.go
- name: List all changed files tracked and untracked files
if: steps.verify-changed-files.outputs.files_changed == 'true'
run: |
echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"
# Fail if files need regeneration
- name: Add Label
if: steps.verify-changed-files.outputs.files_changed == 'true'
uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260
with:
add-labels: 'needs-go-generate-${{matrix.package}}'
repo-token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ needs.issue_number.outputs.issue_number }}
- name: Remove Label
if: steps.verify-changed-files.outputs.files_changed != 'true'
uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260
with:
remove-labels: 'needs-go-generate-${{matrix.package}}'
repo-token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ needs.issue_number.outputs.issue_number }}
check-generation:
name: Go Generate (Module Changes)
runs-on: ubuntu-latest
needs: [changes, issue_number]
if: ${{ github.event_name != 'pull_request' && needs.changes.outputs.package_count_deps > 0 }}
strategy:
fail-fast: false
matrix:
# only do on agents for now. Anything that relies on solidity in a package should do this
package: ${{ fromJSON(needs.changes.outputs.packages_deps) }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
submodules: 'recursive'
- name: Cache Docker images.
uses: ScribeMD/docker-cache@0.2.6
with:
key: docker-generate-${{ runner.os }}-${{ matrix.package }}
- name: Setup NodeJS
uses: ./.github/actions/setup-nodejs
# Foundry is required for flattening
- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1
if: ${{ contains(matrix.package, 'agents') }}
with:
version: nightly
- name: Install Node Dependencies
run: yarn install --immutable
if: ${{ contains(matrix.package, 'agents') }}
# Generate flattened files
- name: Run flattener
run: npx lerna exec npm run build:go
if: ${{ contains(matrix.package, 'agents') }}
# Setup Go
- uses: actions/setup-go@v3
with:
go-version: 1.21.x
- name: Go modules cache
uses: actions/cache@v3
if: ${{ !contains(matrix.package, 'services/cctp-relayer') }}
with:
# see https://github.com/mvdan/github-actions-golang
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
# use seperate cache for generate, builds less stuff
# TODO: consider scoping to package
key: ${{ runner.os }}-go-generate-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-generate-${{matrix.package}}
- name: Cache Linuxbrew
uses: actions/cache@v3
if: ${{ contains(matrix.package, 'scribe') }}
with:
path: |
~/.cache/Homebrew
/home/linuxbrew/.linuxbrew/
key: ${{ runner.os }}-linuxbrew-${{ hashFiles('**/Makefile') }}
# TODO: remove
- name: authenticate with github for private go modules
if: ${{github.event.repository.private}}
uses: fusion-engineering/setup-git-credentials@v2
with:
credentials: https://trajan0x:${{secrets.GIT_TOKEN }}@github.com/
- name: setup env
run: |
echo "::set-env name=GOPATH::$(go env GOPATH)"
echo "::add-path::$(go env GOPATH)/bin"
shell: bash
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
- name: Run Make Generate CI Deps (Scribe)
working-directory: ${{matrix.package}}/
if: ${{ contains(matrix.package, 'scribe') }}
run: |
export PATH=$PATH:$(go env GOPATH)/bin
make generate-ci || exit 0
- name: Run Make Generate CI (Scribe)
working-directory: ${{matrix.package}}/
if: ${{ contains(matrix.package, 'scribe') }}
run: |
make generate-ci
# See if we need to rerun go generate
# TODO: consider implementing https://github.com/golang/go/issues/20520 to speed up process if possible
- name: Try Go Generate
working-directory: ${{matrix.package}}/
if: ${{ !contains(matrix.package, 'services/cctp-relayer') }}
run: |
go generate ./...
env:
GOMEMLIMIT: 6GiB
GOGC: -1
- name: Verify Changed files
uses: tj-actions/verify-changed-files@v11
id: verify-changed-files
with:
files: |
*.go
- name: List all changed files tracked and untracked files
if: steps.verify-changed-files.outputs.files_changed == 'true'
run: |
echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"
# Fail if files need regeneration
- name: Add Label
if: steps.verify-changed-files.outputs.files_changed == 'true'
uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260
with:
add-labels: 'needs-go-generate-${{matrix.package}}'
repo-token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ needs.issue_number.outputs.issue_number }}
- name: Remove Label
if: steps.verify-changed-files.outputs.files_changed != 'true'
uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260
with:
remove-labels: 'needs-go-generate-${{matrix.package}}'
repo-token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ needs.issue_number.outputs.issue_number }}