diff --git a/.github/workflows/check-3rd-party.sh b/.github/workflows/check-3rd-party.sh index 215a930f..112f85fd 100755 --- a/.github/workflows/check-3rd-party.sh +++ b/.github/workflows/check-3rd-party.sh @@ -7,7 +7,11 @@ status=0 for line in `sed -ne 's/[[:space:]-]*uses:[[:space:]]*//p' $1 | sed -e 's/\s*#.*$//'`; do author=`echo $line | awk -F/ '{print $1}'` - if [[ $author == "actions" || $author == "protocol" ]]; then continue; fi + # We trust: + # - .: local workflows + # "actions": workflows authored by GitHub + # "protocol": workflows published in the protocol org + if [[ $author == "." || $author == "actions" || $author == "protocol" ]]; then continue; fi version=`echo $line | awk -F@ '{print $2}' | awk '{print $1}'` if ! [[ "$version" =~ ^[a-f0-9]{40}$ ]]; then status=1 diff --git a/.github/workflows/copy-workflow.yml b/.github/workflows/copy-workflow.yml index 9ed0215c..5cd86ec3 100644 --- a/.github/workflows/copy-workflow.yml +++ b/.github/workflows/copy-workflow.yml @@ -1,24 +1,34 @@ # This workflow is triggered by the dispatch workflow. -on: - repository_dispatch: - types: [ copy-workflow ] - name: Deploy +on: + workflow_dispatch: + inputs: + head_commit_url: + description: "github.event.head_commit.url of the dispatcher" + required: true + files: + description: "List of files to deploy" + required: true + targets: + description: "List of repositories to deploy to" + required: true + jobs: copy: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - cfg: ${{ github.event.client_payload.targets }} + cfg: ${{ fromJson(github.event.inputs.targets) }} env: TARGET_REPO_DIR: "target-repo" TEMPLATE_REPO_DIR: "template-repo" TEMPLATE_DIR: "templates" NEEDS_UPDATE: 0 INITIAL_TEST_DEPLOYMENT: 0 + GO_VERSION_BUMP: 0 GITHUB_USER: "web3-bot" GITHUB_EMAIL: "web3-bot@users.noreply.github.com" name: ${{ matrix.cfg.target }} @@ -38,10 +48,11 @@ jobs: with: # This should be the same Go version we use in the go-check workflow. # go mod tidy, go vet, staticcheck and gofmt might behave differently depending on the version. - go-version: "1.16.x" + stable: 'false' + go-version: "1.17.0-rc1" - name: git config + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR git config user.name ${{ env.GITHUB_USER }} git config user.email ${{ env.GITHUB_EMAIL }} - name: is initial test workflow deployment @@ -50,50 +61,83 @@ jobs: echo "INITIAL_TEST_DEPLOYMENT=1" >> $GITHUB_ENV fi - name: remove the autorebase workflow # TODO: remove this step once all autorebase.yml files have been deleted + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR if [[ -f .github/workflows/autorebase.yml ]]; then git rm .github/workflows/autorebase.yml git commit -m "remove the autorebase workflow" fi - name: remove Travis (on initial deployment) if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }} + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR if [[ -f .travis.yml ]]; then git rm .travis.yml git commit -m "disable Travis" fi - name: remove CircleCI (on initial deployment) if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }} + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR if [[ -d .circleci ]]; then git rm -r .circleci git commit -m "disable CircleCI" fi - name: remove gx (on initial deployment) if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }} + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR if [[ -d .gx ]]; then git rm -r .gx git commit -m "remove .gx" fi - - name: go mod tidy (on initial deployment) - if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }} - run: | - cd $TARGET_REPO_DIR - go mod edit -go 1.15 - go mod tidy - if ! git diff --quiet; then - git add . - git commit -m "set Go version to 1.15 and run go mod tidy" - fi - - name: gofmt (on initial deployment) - if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 }} + - name: bump go.mod go version if needed + uses: protocol/multiple-go-modules@v1.2 + with: + working-directory: ${{ env.TARGET_REPO_DIR }} + run: | + # We want our modules to support two Go versions at a time. + # As of August 2021, Go 1.17 is the latest stable. + # go.mod's Go version declares the language version being used. + # As such, it has to be the minimum of all Go versions supported. + # Bump this every six months, as new Go versions come out. + TARGET_VERSION=1.16 + + # Note that the "<" comparison doesn't understand semver, + # but it should be good enough for the foreseeable future. + CURRENT_VERSION=$(go list -m -f {{.GoVersion}}) + + if [[ $CURRENT_VERSION < $TARGET_VERSION ]]; then + echo "GO_VERSION_BUMP=1" >> $GITHUB_ENV + + # Update the version in go.mod. This alone ensures there's a diff. + go mod edit -go $TARGET_VERSION + + # In the future, "go fix" may make changes to Go code, + # such as to adapt to language changes or API deprecations. + # This is largely a no-op as of Go 1.17, and that's fine. + go fix + git add . + + # We don't tidy, because the next step does that. + # Separate commits also help with reviews. + git commit -m "bump go.mod to Go $TARGET_VERSION and run go fix" + fi + - name: go mod tidy (on initial deployment and on new Go version) + if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}} + uses: protocol/multiple-go-modules@v1.2 + with: + working-directory: ${{ env.TARGET_REPO_DIR }} + run: | + go mod tidy + if ! git diff --quiet; then + git add . + git commit -m "run go mod tidy" + fi + - name: gofmt -s (on initial deployment and on new Go version) + if: ${{ env.INITIAL_TEST_DEPLOYMENT == 1 || env.GO_VERSION_BUMP == 1}} + working-directory: ${{ env.TARGET_REPO_DIR }} run: | - cd $TARGET_REPO_DIR gofmt -s -w . if ! git diff --quiet; then git add . @@ -101,7 +145,7 @@ jobs: fi - name: Add files run: | - for f in $(jq -r ".[]" <<< '${{ toJson(github.event.client_payload.files) }}'); do + for f in $(jq -r ".[]" <<< ${{ toJson(github.event.inputs.files) }}); do echo -e "\nProcessing $f." # add DO NOT EDIT header tmp=$(mktemp) @@ -131,16 +175,15 @@ jobs: popd > /dev/null done - name: Check if we need to create a PR - run: | - cd $TARGET_REPO_DIR - echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count)" >> $GITHUB_ENV + working-directory: ${{ env.TARGET_REPO_DIR }} + run: echo "NEEDS_UPDATE=$(git rev-list HEAD...origin/$(git rev-parse --abbrev-ref HEAD) --ignore-submodules --count)" >> $GITHUB_ENV - name: Create Pull Request if: ${{ env.NEEDS_UPDATE }} - uses: peter-evans/create-pull-request@89a67b1c69299ccc4dcc2253c3e91001adba29b5 # https://github.com/peter-evans/create-pull-request/pull/856 + uses: peter-evans/create-pull-request@a3d36d978a8605d71fdb61c96bf0dfd9c9d8f897 # https://github.com/peter-evans/create-pull-request/pull/856 with: path: ${{ env.TARGET_REPO_DIR }} title: "sync: update CI config files" - body: Syncing to commit ${{ github.event.client_payload.github_event.head_commit.url }}. + body: Syncing to commit ${{ github.event.inputs.head_commit_url }}. token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} committer: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}> author: ${{ env.GITHUB_USER }} <${{ env.GITHUB_EMAIL }}> diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index ad659182..4e693ac3 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -33,11 +33,19 @@ jobs: strategy: fail-fast: false matrix: + # We end up with multiple "dispatch" jobs, + # one per TARGETS "key" chunk above with a "value" array. + # For each "dispatch" job, matrix.cfg.value is an array, like: + # + # [{"target": "repo1", "target": "repo2"}] + # + # The triggered copy-workflow jobs use that final array as their matrix. + # As such, we'll end up with one copy-workflow parallel job per target. cfg: ${{ fromJson(needs.matrix.outputs.targets) }} name: Start copy workflow (batch ${{ matrix.cfg.key }}) steps: - - uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4 # v1.1.3 + - uses: benc-uk/workflow-dispatch@4c044c1613fabbe5250deadc65452d54c4ad4fc7 # v1.1.0 with: + workflow: "Deploy" # "name" attribute of copy-workflow.yml token: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} - event-type: copy-workflow - client-payload: '{ "github_event": ${{ toJson(github.event) }}, "files": ${{ env.FILES }}, "targets": ${{ toJson(matrix.cfg.value) }} }' + inputs: '{ "head_commit_url": ${{ toJson(github.event.head_commit.url) }}, "files": ${{ toJson(env.FILES) }}, "targets": ${{ toJson(toJson(matrix.cfg.value)) }} }' diff --git a/README.md b/README.md index 7c789872..d5a61602 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,43 @@ By storing them in a central place (here), and distributing them in an automated 1. Consistency: Every participating repository uses the same workflow, ensuring that our code adheres to the same coding standards and is properly tested. 2. Maintainability: Workflows change over time. We need to be able to make changes without manually updating dozens of repositories. +## Customization + +### Additional Setup Steps + +Most repositories won't need any customization, and the workflows defined here will just work fine. +Some repositories may require some pre-setup steps to be run before tests (or code checks) can be run. Setup steps for `go-test` are defined in `.github/actions/go-test-setup/action.yml`, and setup steps for `go-check` are defined in `.github/actions/go-check-setup/action.yml`, in the following format: + +```yml +runs: + using: "composite" + steps: + - name: Step 1 + shell: bash + run: echo "do some initial setup" + - name: Step 2 + shell: bash + run: echo "do some Linux-specific setup" + if: ${{ matrix.os == 'ubuntu' }} +``` + +These setup steps are run after the repository has been checked out and after Go has been installed, but before any tests or checks are run. + +### Configuration + +`go-check` contains an optional step that checks that running `go generate` doesn't change any files. +This is useful to make sure that the generated code stays in sync. + +This check will be run in repositories that set `gogenerate` to `true` in `.github/workflows/go-check-config.json`: +```json +{ + "gogenerate": true +} +``` + +Note that depending on the code generators used, it might be necessary to [install those first](#additional-setup-steps). +The generators must also be deterministic, to prevent CI from getting different results each time. + ## Technical Details This repository currently defines two workflows for Go repositories: diff --git a/templates/.github/workflows/automerge.yml b/templates/.github/workflows/automerge.yml index 54d3185a..13da9c15 100644 --- a/templates/.github/workflows/automerge.yml +++ b/templates/.github/workflows/automerge.yml @@ -30,7 +30,9 @@ jobs: automerge: needs: automerge-check runs-on: ubuntu-latest - if: ${{ needs.automerge-check.outputs.status == 'true' }} + # The check for the user is redundant here, as this job depends on the automerge-check job, + # but it prevents this job from spinning up, just to be skipped shortly after. + if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true' steps: - name: Wait on tests uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2 diff --git a/templates/.github/workflows/go-check.yml b/templates/.github/workflows/go-check.yml index 1d111fe2..2cb5dd83 100644 --- a/templates/.github/workflows/go-check.yml +++ b/templates/.github/workflows/go-check.yml @@ -5,17 +5,28 @@ jobs: unit: runs-on: ubuntu-latest name: All + env: + RUNGOGENERATE: false steps: - uses: actions/checkout@v2 with: submodules: recursive - uses: actions/setup-go@v2 with: - go-version: "1.16.x" + go-version: "1.17.x" + - name: Run repo-specific setup + uses: ./.github/actions/go-check-setup + if: hashFiles('./.github/actions/go-check-setup') != '' + - name: Read config + if: hashFiles('./.github/workflows/go-check-config.json') != '' + run: | + if jq -re .gogenerate ./.github/workflows/go-check-config.json; then + echo "RUNGOGENERATE=true" >> $GITHUB_ENV + fi - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@434f5f3816b358fe468fa83dcba62d794e7fe04b # 2021.1 (v0.2.0) + run: go install honnef.co/go/tools/cmd/staticcheck@df71e5d0e0ed317ebf43e6e59cf919430fa4b8f2 # 2021.1.1 (v0.2.1) - name: Check that go.mod is tidy - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 with: run: | go mod tidy @@ -34,14 +45,27 @@ jobs: fi - name: go vet if: ${{ success() || failure() }} # run this step even if the previous one failed - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 with: run: go vet ./... - name: staticcheck if: ${{ success() || failure() }} # run this step even if the previous one failed - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 with: run: | set -o pipefail staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g' + - name: go generate + uses: protocol/multiple-go-modules@v1.2 + if: (success() || failure()) && env.RUNGOGENERATE == 'true' + with: + run: | + git clean -fd # make sure there aren't untracked files / directories + go generate ./... + # check if go generate modified or added any files + if ! $(git add . && git diff-index HEAD --exit-code --quiet); then + echo "go generated caused changes to the repository:" + git status --short + exit 1 + fi diff --git a/templates/.github/workflows/go-test.yml b/templates/.github/workflows/go-test.yml index ad61a04a..ceca1cb5 100644 --- a/templates/.github/workflows/go-test.yml +++ b/templates/.github/workflows/go-test.yml @@ -7,7 +7,9 @@ jobs: fail-fast: false matrix: os: [ "ubuntu", "windows", "macos" ] - go: [ "1.15.x", "1.16.x" ] + go: [ "1.16.x", "1.17.x" ] + env: + COVERAGES: "" runs-on: ${{ matrix.os }}-latest name: ${{ matrix.os}} (go ${{ matrix.go }}) steps: @@ -21,24 +23,30 @@ jobs: run: | go version go env + - name: Run repo-specific setup + uses: ./.github/actions/go-test-setup + if: hashFiles('./.github/actions/go-test-setup') != '' - name: Run tests - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 with: - run: go test -v -coverprofile coverage.txt ./... + run: go test -v -coverprofile module-coverage.txt ./... - name: Run tests (32 bit) if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX. - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 env: GOARCH: 386 with: run: go test -v ./... - name: Run tests with race detector if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow - uses: protocol/multiple-go-modules@v1.0 + uses: protocol/multiple-go-modules@v1.2 with: run: go test -v -race ./... + - name: Collect coverage files + shell: bash + run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV - name: Upload coverage to Codecov - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 # v1.5.0 + uses: codecov/codecov-action@51d810878be5422784e86451c0e7c14e5860ec47 # v2.0.2 with: - file: coverage.txt + files: '${{ env.COVERAGES }}' env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}