Skip to content

Commit

Permalink
internal/ci/github: rename repo -> _repo
Browse files Browse the repository at this point in the history 
Running cue export on the github package is a good way of tracking down
issues with workflow declarations that are otherwise impossible to find
when running cue cmd to regenerate the yml outputs. These bugs are known
about and effectively captured by #1325.

Right now, because we declare the field named base to be the package
value of the imported base package, such an export fails because there
are many fields of that package value that cannot be exported because
they are not concrete.

Fix that by making the field hidden, which allows the cue export trick
to work again.

Signed-off-by: Paul Jolly <paul@myitcv.io>
Change-Id: Icd9df2beab51ff2ae34ac3eb641d6c739b01e2ce
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/551826
Unity-Result: CUEcueckoo <cueckoo@cuelang.org>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
  • Loading branch information
@myitcv
myitcv committed Mar 28, 2023
1 parent 7aec30c commit 646e965
Show file tree
Hide file tree
Showing 8 changed files with 63 additions and 63 deletions.
22 changes: 11 additions & 11 deletions internal/ci/github/evict_caches.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
)

// The evict_caches removes "old" GitHub actions caches from the main repo and
// the accompanying trybot repo. The job is only run in the main repo, because
// the accompanying trybot _repo. The job is only run in the main repo, because
// that is the only place where the credentials exist.
//
// The GitHub actions caches in the main and trybot repos can get large. So
Expand All @@ -40,7 +40,7 @@ import (
//
// In testing with @mvdan, this resulted in cache sizes for Linux dropping from
// ~1GB to ~125MB. This is a considerable saving.
workflows: evict_caches: repo.bashWorkflow & {
workflows: evict_caches: _repo.bashWorkflow & {
name: "Evict caches"

on: {
Expand All @@ -52,20 +52,20 @@ workflows: evict_caches: repo.bashWorkflow & {
jobs: {
test: {
// We only want to run this in the main repo
if: "${{github.repository == '\(repo.githubRepositoryPath)'}}"
"runs-on": repo.linuxMachine
if: "${{github.repository == '\(_repo.githubRepositoryPath)'}}"
"runs-on": _repo.linuxMachine
steps: [
json.#step & {
let branchPatterns = strings.Join(repo.protectedBranchPatterns, " ")
let branchPatterns = strings.Join(_repo.protectedBranchPatterns, " ")

// rerunLatestWorkflow runs the latest trybot workflow in the
// specified repo for branches that match the specified branch.
let rerunLatestWorkflow = {
#repo: string
#branch: string
"""
id=$(\(repo.curlGitHubAPI) "https://github.com/gitapi/repos/\(#repo)/actions/workflows/trybot.yml/runs?branch=\(#branch)&event=push&per_page=1" | jq '.workflow_runs[] | .id')
\(repo.curlGitHubAPI) -X POST https://github.com/gitapi/repos/\(#repo)/actions/runs/$id/rerun
id=$(\(_repo.curlGitHubAPI) "https://github.com/gitapi/repos/\(#repo)/actions/workflows/trybot.yml/runs?branch=\(#branch)&event=push&per_page=1" | jq '.workflow_runs[] | .id')
\(_repo.curlGitHubAPI) -X POST https://github.com/gitapi/repos/\(#repo)/actions/runs/$id/rerun
"""
}
Expand All @@ -75,7 +75,7 @@ workflows: evict_caches: repo.bashWorkflow & {
echo ${{ secrets.CUECKOO_GITHUB_PAT }} | gh auth login --with-token
gh extension install actions/gh-actions-cache
for i in \(repo.githubRepositoryURL) \(repo.githubRepositoryURL)-trybot
for i in \(_repo.githubRepositoryURL) \(_repo.githubRepositoryURL)-trybot
do
echo "Evicting caches for $i"
cd $(mktemp -d)
Expand All @@ -90,7 +90,7 @@ workflows: evict_caches: repo.bashWorkflow & {
# Now trigger the most recent workflow run on each of the default branches.
# We do this by listing all the branches on the main repo and finding those
# which match the protected branch patterns (globs).
for j in $(\(repo.curlGitHubAPI) -f https://github.com/gitapi/repos/\(repo.githubRepositoryPath)/branches | jq -r '.[] | .name')
for j in $(\(_repo.curlGitHubAPI) -f https://github.com/gitapi/repos/\(_repo.githubRepositoryPath)/branches | jq -r '.[] | .name')
do
for i in \(branchPatterns)
do
Expand All @@ -99,8 +99,8 @@ workflows: evict_caches: repo.bashWorkflow & {
fi
echo "$j is a match with $i"
\(rerunLatestWorkflow & {#repo: repo.githubRepositoryPath, #branch: "$j", _})
\(rerunLatestWorkflow & {#repo: repo.githubRepositoryPath + "-trybot", #branch: "$j", _})
\(rerunLatestWorkflow & {#repo: _repo.githubRepositoryPath, #branch: "$j", _})
\(rerunLatestWorkflow & {#repo: _repo.githubRepositoryPath + "-trybot", #branch: "$j", _})
done
done
"""
Expand Down
10 changes: 5 additions & 5 deletions internal/ci/github/push_tip_to_trybot.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@

package github

// push_tip_to_trybot "syncs" active branches to the trybot repo.
// push_tip_to_trybot "syncs" active branches to the trybot _repo.
// Since the workflow is triggered by a push to any of the branches,
// the step only needs to sync the pushed branch.
workflows: push_tip_to_trybot: repo.pushTipToTrybotWorkflow & {
workflows: push_tip_to_trybot: _repo.pushTipToTrybotWorkflow & {
on: {
push: branches: repo.protectedBranchPatterns
push: branches: _repo.protectedBranchPatterns
}
jobs: push: {
"runs-on": repo.linuxMachine
if: "${{github.repository == '\(repo.githubRepositoryPath)'}}"
"runs-on": _repo.linuxMachine
if: "${{github.repository == '\(_repo.githubRepositoryPath)'}}"
}
}
30 changes: 15 additions & 15 deletions internal/ci/github/release.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
_cueVersionRef: "${GITHUB_REF##refs/tags/}"

// The release workflow
workflows: release: repo.bashWorkflow & {
workflows: release: _repo.bashWorkflow & {

name: "Release"

Expand All @@ -36,16 +36,16 @@ workflows: release: repo.bashWorkflow & {
concurrency: "release"

on: push: {
tags: [repo.releaseTagPattern, "!" + repo.zeroReleaseTagPattern]
branches: list.Concat([[repo.testDefaultBranch], repo.protectedBranchPatterns])
tags: [_repo.releaseTagPattern, "!" + _repo.zeroReleaseTagPattern]
branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns])
}
jobs: goreleaser: {
"runs-on": repo.linuxMachine
if: "${{github.repository == '\(repo.githubRepositoryPath)'}}"
"runs-on": _repo.linuxMachine
if: "${{github.repository == '\(_repo.githubRepositoryPath)'}}"
steps: [
for v in repo.checkoutCode {v},
repo.installGo & {
with: "go-version": repo.pinnedReleaseGo
for v in _repo.checkoutCode {v},
_repo.installGo & {
with: "go-version": _repo.pinnedReleaseGo
},
json.#step & {
name: "Setup qemu"
Expand Down Expand Up @@ -73,7 +73,7 @@ workflows: release: repo.bashWorkflow & {
uses: "goreleaser/goreleaser-action@v3"
with: {
"install-only": true
version: repo.goreleaserVersion
version: _repo.goreleaserVersion
}
},
json.#step & {
Expand All @@ -84,18 +84,18 @@ workflows: release: repo.bashWorkflow & {
run: "cue cmd release"
"working-directory": "./internal/ci/goreleaser"
},
repo.repositoryDispatch & {
_repo.repositoryDispatch & {
name: "Re-test cuelang.org"
if: repo.isReleaseTag
#githubRepositoryPath: repo.cuelangRepositoryPath
if: _repo.isReleaseTag
#githubRepositoryPath: _repo.cuelangRepositoryPath
#arg: {
event_type: "Re-test post release of \(_cueVersionRef)"
}
},
repo.repositoryDispatch & {
_repo.repositoryDispatch & {
name: "Trigger unity build"
if: repo.isReleaseTag
#githubRepositoryPath: repo.unityRepositoryPath
if: _repo.isReleaseTag
#githubRepositoryPath: _repo.unityRepositoryPath
#arg: {
event_type: "Check against CUE \(_cueVersionRef)"
client_payload: {
Expand Down
4 changes: 2 additions & 2 deletions internal/ci/github/repo.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ package github
// in a single file, and that keeps the different in import
// path down to a single file.

import _repo "cuelang.org/go/internal/ci/repo"
import repo "cuelang.org/go/internal/ci/repo"

repo: _repo
_repo: repo
16 changes: 8 additions & 8 deletions internal/ci/github/tip_triggers.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,27 @@ package github

// The tip_triggers workflow. This fires for each new commit that hits the
// default branch.
workflows: tip_triggers: repo.bashWorkflow & {
workflows: tip_triggers: _repo.bashWorkflow & {

name: "Triggers on push to tip"
on: push: branches: [repo.defaultBranch]
on: push: branches: [_repo.defaultBranch]
jobs: push: {
"runs-on": repo.linuxMachine
if: "${{github.repository == '\(repo.githubRepositoryPath)'}}"
"runs-on": _repo.linuxMachine
if: "${{github.repository == '\(_repo.githubRepositoryPath)'}}"
steps: [
repo.repositoryDispatch & {
_repo.repositoryDispatch & {
name: "Trigger tip.cuelang.org deploy"
#githubRepositoryPath: repo.cuelangRepositoryPath
#githubRepositoryPath: _repo.cuelangRepositoryPath
#arg: {
event_type: "Rebuild tip against ${GITHUB_SHA}"
client_payload: {
type: "rebuild_tip"
}
}
},
repo.repositoryDispatch & {
_repo.repositoryDispatch & {
name: "Trigger unity build"
#githubRepositoryPath: repo.unityRepositoryPath
#githubRepositoryPath: _repo.unityRepositoryPath
#arg: {
event_type: "Check against ${GITHUB_SHA}"
client_payload: {
Expand Down
40 changes: 20 additions & 20 deletions internal/ci/github/trybot.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ import (
)

// The trybot workflow.
workflows: trybot: repo.bashWorkflow & {
name: repo.trybot.name
workflows: trybot: _repo.bashWorkflow & {
name: _repo.trybot.name

on: {
push: {
branches: list.Concat([["trybot/*/*", repo.testDefaultBranch], repo.protectedBranchPatterns]) // do not run PR branches
"tags-ignore": [repo.releaseTagPattern]
branches: list.Concat([["trybot/*/*", _repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches
"tags-ignore": [_repo.releaseTagPattern]
}
pull_request: {}
}
Expand All @@ -37,11 +37,11 @@ workflows: trybot: repo.bashWorkflow & {
strategy: _testStrategy
"runs-on": "${{ matrix.os }}"

let goCaches = repo.setupGoActionsCaches & {#protectedBranchExpr: repo.isProtectedBranch, _}
let goCaches = _repo.setupGoActionsCaches & {#protectedBranchExpr: _repo.isProtectedBranch, _}

steps: [
for v in repo.checkoutCode {v},
repo.installGo,
for v in _repo.checkoutCode {v},
_repo.installGo,

// cachePre must come after installing Node and Go, because the cache locations
// are established by running each tool.
Expand All @@ -53,28 +53,28 @@ workflows: trybot: repo.bashWorkflow & {
// subsequent CLs in the trybot repo can leverage the updated
// cache. Therefore, we instead perform a clean of the testcache.
json.#step & {
if: "github.repository == '\(repo.githubRepositoryPath)' && (\(repo.isProtectedBranch) || github.ref == 'refs/heads/\(repo.testDefaultBranch)')"
if: "github.repository == '\(_repo.githubRepositoryPath)' && (\(_repo.isProtectedBranch) || github.ref == 'refs/heads/\(_repo.testDefaultBranch)')"
run: "go clean -testcache"
},

repo.earlyChecks & {
_repo.earlyChecks & {
// These checks don't vary based on the Go version or OS,
// so we only need to run them on one of the matrix jobs.
if: repo.isLatestLinux
if: _repo.isLatestLinux
},
json.#step & {
if: "\(repo.isProtectedBranch) || \(repo.isLatestLinux)"
if: "\(_repo.isProtectedBranch) || \(_repo.isLatestLinux)"
run: "echo CUE_LONG=true >> $GITHUB_ENV"
},
_goGenerate,
_goTest & {
if: "\(repo.isProtectedBranch) || !\(repo.isLatestLinux)"
if: "\(_repo.isProtectedBranch) || !\(_repo.isLatestLinux)"
},
_goTestRace & {
if: repo.isLatestLinux
if: _repo.isLatestLinux
},
_goCheck,
repo.checkGitClean,
_repo.checkGitClean,
_pullThroughProxy,
]
}
Expand All @@ -83,13 +83,13 @@ workflows: trybot: repo.bashWorkflow & {
_testStrategy: {
"fail-fast": false
matrix: {
"go-version": ["1.19.x", repo.latestStableGo]
os: [repo.linuxMachine, repo.macosMachine, repo.windowsMachine]
"go-version": ["1.19.x", _repo.latestStableGo]
os: [_repo.linuxMachine, _repo.macosMachine, _repo.windowsMachine]
}
}

_pullThroughProxy: json.#step & {
name: "Pull this commit through the proxy on \(repo.defaultBranch)"
name: "Pull this commit through the proxy on \(_repo.defaultBranch)"
run: """
v=$(git rev-parse HEAD)
cd $(mktemp -d)
Expand Down Expand Up @@ -121,15 +121,15 @@ workflows: trybot: repo.bashWorkflow & {
echo "giving up after a number of retries"
exit 1
"""
if: "\(repo.isProtectedBranch) && \(repo.isLatestLinux)"
if: "\(_repo.isProtectedBranch) && \(_repo.isLatestLinux)"
}

_goGenerate: json.#step & {
name: "Generate"
run: "go generate ./..."
// The Go version corresponds to the precise version specified in
// the matrix. Skip windows for now until we work out why re-gen is flaky
if: repo.isLatestLinux
if: _repo.isLatestLinux
}

_goTest: json.#step & {
Expand All @@ -144,7 +144,7 @@ workflows: trybot: repo.bashWorkflow & {
// dependencies that vary wildly between platforms.
// For now, to save CI resources, just run the checks on one matrix job.
// TODO: consider adding more checks as per https://github.com/golang/go/issues/42119.
if: "\(repo.isLatestLinux)"
if: "\(_repo.isLatestLinux)"
name: "Check"
run: "go vet ./..."
}
Expand Down
2 changes: 1 addition & 1 deletion internal/ci/github/trybot_dispatch.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@
package github

// The trybot_dispatch workflow.
workflows: trybot_dispatch: repo.bashWorkflow & repo.trybotDispatchWorkflow
workflows: trybot_dispatch: _repo.bashWorkflow & _repo.trybotDispatchWorkflow
2 changes: 1 addition & 1 deletion internal/ci/github/workflows.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
workflows: close({
[string]: json.#Workflow

(repo.trybot.key): _
(_repo.trybot.key): _
trybot_dispatch: _
release: _
tip_triggers: _
Expand Down

0 comments on commit 646e965

Please sign in to comment.