From e9395854e756c567eeea6434df66eb545cac5410 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 4 Dec 2020 18:23:17 +0100 Subject: [PATCH 01/21] Use credential bindings for osrfbuild github token --- jenkins-scripts/dsl/brew_release.dsl | 14 +++++++++++--- jenkins-scripts/lib/_homebrew_github_setup.bash | 12 ------------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 71d5d6be8..04153bee6 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -75,6 +75,10 @@ release_job.with 'Version of the package just released') stringParam('SOURCE_TARBALL_SHA','', 'SHA Hash of the tarball file') + credentialsBinding { + // credential name needs to be in sync with provision code at infra/osrf-chef repo + string('GITHUB_TOKEN', 'osrfbuild-token') + } } steps @@ -254,9 +258,13 @@ bottle_job_hash_updater.with parameters { - // reuse the pull request created by homebrew_pull_request_updater in step 1 - stringParam("PULL_REQUEST_URL", '', - 'Pull request URL (osrf/homebrew-simulation) pointing to a pull request.') + // reuse the pull request created by homebrew_pull_request_updater in step 1 + stringParam("PULL_REQUEST_URL", '', + 'Pull request URL (osrf/homebrew-simulation) pointing to a pull request.') + credentialsBinding { + // credential name needs to be in sync with provision code at infra/osrf-chef repo + string('GITHUB_TOKEN', 'osrfbuild-token') + } } steps diff --git a/jenkins-scripts/lib/_homebrew_github_setup.bash b/jenkins-scripts/lib/_homebrew_github_setup.bash index af3e5ce1a..a96276af2 100644 --- a/jenkins-scripts/lib/_homebrew_github_setup.bash +++ b/jenkins-scripts/lib/_homebrew_github_setup.bash @@ -22,18 +22,6 @@ if [[ -z $(ssh -T git@github.com 2>&1 | grep successfully) ]]; then exit 1 fi -GITHUB_TOKEN_FILE="/var/lib/jenkins/.github_token" -if [[ ! -f ${GITHUB_TOKEN_FILE} ]]; then - echo "The hub cli tool needs a valid token at file ${GITHUB_TOKEN_FILE}" - echo "The file was not found" - exit 1 -fi - -set +x # keep password secret -export GITHUB_TOKEN=`cat $GITHUB_TOKEN_FILE` -set -x # back to debug -echo '# END SECTION' - echo '# BEGIN SECTION: download linuxbrew' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" echo '# END SECTION' From f2a26b03e082573d14b2c9f7a88e9b41960588ed Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:32:57 +0100 Subject: [PATCH 02/21] Refactor Github Credential bindings to use in test job --- .../GitHubCredentialOsrfbuild.groovy | 19 +++++++++++++++++++ jenkins-scripts/dsl/brew_release.dsl | 5 +---- jenkins-scripts/dsl/test.dsl | 17 +++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy new file mode 100644 index 000000000..0a760a67e --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -0,0 +1,19 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job + +class GitHubCredentialOsrfbuild +{ + static void create(Job job) + { + job.with + { + parameters { + // credential name needs to be in sync with provision code at infra/osrf-chef repo + credentialsBinding { + string('GITHUB_TOKEN', 'osrfbuild-token') + } + } + } + } +} diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 04153bee6..e5c031cd4 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -245,6 +245,7 @@ bottle_job_builder.with def bottle_job_hash_updater = job(bottle_hash_updater_job_name) OSRFUNIXBase.create(bottle_job_hash_updater) GenericRemoteToken.create(bottle_job_hash_updater) +GitHubCredentialOsrfbuild.create(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with @@ -261,10 +262,6 @@ bottle_job_hash_updater.with // reuse the pull request created by homebrew_pull_request_updater in step 1 stringParam("PULL_REQUEST_URL", '', 'Pull request URL (osrf/homebrew-simulation) pointing to a pull request.') - credentialsBinding { - // credential name needs to be in sync with provision code at infra/osrf-chef repo - string('GITHUB_TOKEN', 'osrfbuild-token') - } } steps diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 5f9d809df..776244932 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -15,3 +15,20 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, false, ['main']) +def test_credentials_job = job("_test_credentials_from_dsl") +OSRFLinuxBase.create(test_credentials_job) +GitHubCredentialOsrfbuild.create(test_credentials_job) + +test_credentials_job.with +{ + steps { + shell("""\ + #!/bin/bash -xe + + export ssh_log=`ssh -T git@github.com 2>&1` + echo \$ssl_log + grep osrf-jenkins <<< \$ssh_log || exit 1 + """.stripIndent()) + } + } +} From 6b1093108ae4d9eb1f4773b92e00a4c776ef4591 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:35:18 +0100 Subject: [PATCH 03/21] Fix syntax Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 776244932..863afe501 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -30,5 +30,4 @@ test_credentials_job.with grep osrf-jenkins <<< \$ssh_log || exit 1 """.stripIndent()) } - } } From bc5e095b205a4ae497d3dea5fe41a5fe165cc699 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:38:17 +0100 Subject: [PATCH 04/21] crendtialBindings work on wrappers context Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy index 0a760a67e..d82508884 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -8,7 +8,7 @@ class GitHubCredentialOsrfbuild { job.with { - parameters { + wrappers { // credential name needs to be in sync with provision code at infra/osrf-chef repo credentialsBinding { string('GITHUB_TOKEN', 'osrfbuild-token') From 8a346e4e256bb40a3a4f51d4110a80f852d92ad8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:39:54 +0100 Subject: [PATCH 05/21] Fix username Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 863afe501..c175cc511 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -27,7 +27,7 @@ test_credentials_job.with export ssh_log=`ssh -T git@github.com 2>&1` echo \$ssl_log - grep osrf-jenkins <<< \$ssh_log || exit 1 + grep osrfbuild <<< \$ssh_log || exit 1 """.stripIndent()) } } From 2a446d37f202c469e8097edb2dc3f5de13dafff0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:42:24 +0100 Subject: [PATCH 06/21] Use lower class Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index c175cc511..4ce8f3f8a 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -16,7 +16,7 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, ['main']) def test_credentials_job = job("_test_credentials_from_dsl") -OSRFLinuxBase.create(test_credentials_job) +OSRFBase.create(test_credentials_job) GitHubCredentialOsrfbuild.create(test_credentials_job) test_credentials_job.with From 0bd6790785ea82170b262a1a13142e54281abf97 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 18:43:08 +0100 Subject: [PATCH 07/21] Change job name Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/test.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 4ce8f3f8a..82cfb3d37 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -15,7 +15,7 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, false, ['main']) -def test_credentials_job = job("_test_credentials_from_dsl") +def test_credentials_job = job("_test_job_osrfbuild_credentials_from_dsl") OSRFBase.create(test_credentials_job) GitHubCredentialOsrfbuild.create(test_credentials_job) From cbbf481a7796eaaf902a8559d93fa5eeedd34360 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 19:53:26 +0100 Subject: [PATCH 08/21] Use crendentials for ssh keys --- .../GitHubCredentialOsrfbuild.groovy | 22 +++++++++++++++- jenkins-scripts/dsl/test.dsl | 26 +++++++++++++++---- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy index d82508884..e5e8694aa 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job class GitHubCredentialOsrfbuild { - static void create(Job job) + static void createToken(Job job) { job.with { @@ -16,4 +16,24 @@ class GitHubCredentialOsrfbuild } } } + + static void createKey(Job job) + { + job.with + { + configure { project -> + project / wrappers /'org.jenkinsci.plugins.credentialsbinding.impl.SecretBuildWrapper' { + 'bindings' { + 'org.jenkinsci.plugins.credentialsbinding.impl.SSHUserPrivateKeyBinding' { + // credentialId needs to be in sync with provision code at infra/osrf-chef repo + credentialsId 'osrfbuild' + keyFileVariable 'OSRFBUILD_KEY' + usernameVariable() + passphraseVariable() + } + } + } + } + } + } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 82cfb3d37..93e034201 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -15,11 +15,10 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, false, false, ['main']) -def test_credentials_job = job("_test_job_osrfbuild_credentials_from_dsl") -OSRFBase.create(test_credentials_job) -GitHubCredentialOsrfbuild.create(test_credentials_job) - -test_credentials_job.with +def test_credentials_token_job = job("_test_job_osrfbuild-credentials-token_from_dsl") +OSRFBase.create(test_credentials_token_job) +GitHubCredentialOsrfbuild.createToken(test_credentials_token_job) +test_credentials_token_job.with { steps { shell("""\ @@ -31,3 +30,20 @@ test_credentials_job.with """.stripIndent()) } } + +def test_credentials_key_job = job("_test_job_osrfbuild-credentials-key_from_dsl") +OSRFBase.create(test_credentials_key_job) +GitHubCredentialOsrfbuild.createKey(test_credentials_key_job) + +test_credentials_key_job.with +{ + steps { + shell("""\ + #!/bin/bash -xe + + export ssh_log=`ssh -T -i \${OSRFBUILD_KEY} git@github.com 2>&1` + echo \$ssl_log + grep osrfbuild <<< \$ssh_log || exit 1 + """.stripIndent()) + } +} From 733e4d1c91c3839ab6ba44b150b34a472f1534f0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 20:01:33 +0100 Subject: [PATCH 09/21] Use ssh-agent Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/GitHubCredentialOsrfbuild.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy index e5e8694aa..5dd676351 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -21,6 +21,11 @@ class GitHubCredentialOsrfbuild { job.with { + wrappers { + sshAgent('osrfbuild') + } + } +/* configure { project -> project / wrappers /'org.jenkinsci.plugins.credentialsbinding.impl.SecretBuildWrapper' { 'bindings' { @@ -36,4 +41,5 @@ class GitHubCredentialOsrfbuild } } } +*/ } From 0808e841956d16131aca71d4a1eb43998563c928 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 20:03:57 +0100 Subject: [PATCH 10/21] Fix syntax Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy index 5dd676351..ca5113f90 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -40,6 +40,6 @@ class GitHubCredentialOsrfbuild } } } - } */ + } } From 56e83f223d9de34e10bcbcf4e6be663cbd052121 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 23 Feb 2021 20:44:05 +0100 Subject: [PATCH 11/21] Key management should not be needed if using token --- .../GitHubCredentialOsrfbuild.groovy | 30 ++----------------- jenkins-scripts/dsl/test.dsl | 17 ----------- 2 files changed, 2 insertions(+), 45 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy index ca5113f90..cb5bc374e 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy @@ -2,9 +2,9 @@ package _configs_ import javaposse.jobdsl.dsl.Job -class GitHubCredentialOsrfbuild +class GitHubCredentials { - static void createToken(Job job) + static void createOsrfbuildToken(Job job) { job.with { @@ -16,30 +16,4 @@ class GitHubCredentialOsrfbuild } } } - - static void createKey(Job job) - { - job.with - { - wrappers { - sshAgent('osrfbuild') - } - } -/* - configure { project -> - project / wrappers /'org.jenkinsci.plugins.credentialsbinding.impl.SecretBuildWrapper' { - 'bindings' { - 'org.jenkinsci.plugins.credentialsbinding.impl.SSHUserPrivateKeyBinding' { - // credentialId needs to be in sync with provision code at infra/osrf-chef repo - credentialsId 'osrfbuild' - keyFileVariable 'OSRFBUILD_KEY' - usernameVariable() - passphraseVariable() - } - } - } - } - } -*/ - } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 93e034201..dc32212f4 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -30,20 +30,3 @@ test_credentials_token_job.with """.stripIndent()) } } - -def test_credentials_key_job = job("_test_job_osrfbuild-credentials-key_from_dsl") -OSRFBase.create(test_credentials_key_job) -GitHubCredentialOsrfbuild.createKey(test_credentials_key_job) - -test_credentials_key_job.with -{ - steps { - shell("""\ - #!/bin/bash -xe - - export ssh_log=`ssh -T -i \${OSRFBUILD_KEY} git@github.com 2>&1` - echo \$ssl_log - grep osrfbuild <<< \$ssh_log || exit 1 - """.stripIndent()) - } -} From cf1a678cf00474a703bffdd969bfc19513b01158 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Feb 2021 15:35:02 +0100 Subject: [PATCH 12/21] Move token to auxiliary class --- ...ntialOsrfbuild.groovy => GitHubCredentials.groovy} | 0 jenkins-scripts/dsl/brew_release.dsl | 11 ++--------- jenkins-scripts/dsl/test.dsl | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) rename jenkins-scripts/dsl/_configs_/{GitHubCredentialOsrfbuild.groovy => GitHubCredentials.groovy} (100%) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy similarity index 100% rename from jenkins-scripts/dsl/_configs_/GitHubCredentialOsrfbuild.groovy rename to jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index e5c031cd4..52c9a2d53 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -52,6 +52,7 @@ void include_common_params(Job job) def release_job = job("generic-release-homebrew_pull_request_updater") OSRFUNIXBase.create(release_job) GenericRemoteToken.create(release_job) +GitHubCredentials.createOsrfbuildToken(release_job) include_common_params(release_job) release_job.with @@ -75,10 +76,6 @@ release_job.with 'Version of the package just released') stringParam('SOURCE_TARBALL_SHA','', 'SHA Hash of the tarball file') - credentialsBinding { - // credential name needs to be in sync with provision code at infra/osrf-chef repo - string('GITHUB_TOKEN', 'osrfbuild-token') - } } steps @@ -137,10 +134,6 @@ bottle_job_builder.with { wrappers { preBuildCleanup() - credentialsBinding { - // crendetial name needs to be in sync with provision code at infra/osrf-chef repo - string('GITHUB_TOKEN', 'osrf-migration-token') - } } properties { @@ -245,7 +238,7 @@ bottle_job_builder.with def bottle_job_hash_updater = job(bottle_hash_updater_job_name) OSRFUNIXBase.create(bottle_job_hash_updater) GenericRemoteToken.create(bottle_job_hash_updater) -GitHubCredentialOsrfbuild.create(bottle_job_hash_updater) +GitHubCredentials.createOsrfbuildToken(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index dc32212f4..83c57935a 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -17,7 +17,7 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, ['main']) def test_credentials_token_job = job("_test_job_osrfbuild-credentials-token_from_dsl") OSRFBase.create(test_credentials_token_job) -GitHubCredentialOsrfbuild.createToken(test_credentials_token_job) +GitHubCredentials.createOsrfbuildToken(test_credentials_token_job) test_credentials_token_job.with { steps { From 69bd5c14a11d08f6ea4ee9d1e04cd08f6d0c5c3a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Feb 2021 15:42:55 +0100 Subject: [PATCH 13/21] Implement API check for permissions --- jenkins-scripts/dsl/test.dsl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 83c57935a..b0c0c2bf4 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -27,6 +27,14 @@ test_credentials_token_job.with export ssh_log=`ssh -T git@github.com 2>&1` echo \$ssl_log grep osrfbuild <<< \$ssh_log || exit 1 + + # keep passowrd secret + set +x + curl -u osrfbuild:\$GITHUB_TOKEN \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/osrf/homebrew-simulation/collaborators/osrfbuild/permission > github_check_log + set -x + """.stripIndent()) } } From 20551f347e62987d1af199835e4c7384ffe8fa92 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Feb 2021 15:54:42 +0100 Subject: [PATCH 14/21] Check for permissions using a temporary branch --- jenkins-scripts/dsl/test.dsl | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index b0c0c2bf4..77516a304 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -28,13 +28,22 @@ test_credentials_token_job.with echo \$ssl_log grep osrfbuild <<< \$ssh_log || exit 1 - # keep passowrd secret - set +x - curl -u osrfbuild:\$GITHUB_TOKEN \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/osrf/homebrew-simulation/collaborators/osrfbuild/permission > github_check_log - set -x + # Check push+commit permissions for osrfbuild by uploading/deleting a + # branch. Note that call to the API for permissions require of admin + # perms that osrfbuild user does not have. + git clone git@github.com:osrf/homebrew-simulation.git + cd homebrew-simulation + git config user.name 'osrfbuild' --replace-all + git config user.email 'osrfbuild@openrobotics.org' --replace-all + git checkout -b _test_job_osrfbuild_ + git commit --allow-empty -m "testing commit" + git push -u origin _test_job_osrfbuild_ + git push origin --delete _test_job_osrfbuild_ """.stripIndent()) } + + wrappers { + preBuildCleanup() + } } From cc1cc08abc8661286b254be5c56221ee275aab9f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Feb 2021 18:05:34 +0100 Subject: [PATCH 15/21] Run the job on Linux --- jenkins-scripts/dsl/test.dsl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 77516a304..f8bcf7f82 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -20,6 +20,8 @@ OSRFBase.create(test_credentials_token_job) GitHubCredentials.createOsrfbuildToken(test_credentials_token_job) test_credentials_token_job.with { + label "docker" + steps { shell("""\ #!/bin/bash -xe From 2c5ac8a5fe541c6b7334ba7ec7d7223a7fa7e6e9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 25 Feb 2021 20:48:49 +0100 Subject: [PATCH 16/21] Update test --- jenkins-scripts/dsl/test.dsl | 19 ++++++++++--------- .../lib/_homebrew_github_setup.bash | 13 ++----------- .../lib/homebrew_formula_pullrequest.bash | 2 +- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index f8bcf7f82..d7bc3d2a5 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -20,28 +20,29 @@ OSRFBase.create(test_credentials_token_job) GitHubCredentials.createOsrfbuildToken(test_credentials_token_job) test_credentials_token_job.with { - label "docker" + label "osx" steps { shell("""\ #!/bin/bash -xe - export ssh_log=`ssh -T git@github.com 2>&1` - echo \$ssl_log - grep osrfbuild <<< \$ssh_log || exit 1 - # Check push+commit permissions for osrfbuild by uploading/deleting a # branch. Note that call to the API for permissions require of admin - # perms that osrfbuild user does not have. + # perms that osrfbuild user does not have. Personal tokens don't + # support ssh but https only. - git clone git@github.com:osrf/homebrew-simulation.git + git clone https://github.com/osrfbuild/homebrew-simulation.git cd homebrew-simulation git config user.name 'osrfbuild' --replace-all git config user.email 'osrfbuild@openrobotics.org' --replace-all + set +x + git config url."https://osrfbuild:\${GITHUB_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git + set -x git checkout -b _test_job_osrfbuild_ git commit --allow-empty -m "testing commit" - git push -u origin _test_job_osrfbuild_ - git push origin --delete _test_job_osrfbuild_ + # protect token from errors + git push -u origin _test_job_osrfbuild_ > push_log + git push origin --delete _test_job_osrfbuild_ >> push_log """.stripIndent()) } diff --git a/jenkins-scripts/lib/_homebrew_github_setup.bash b/jenkins-scripts/lib/_homebrew_github_setup.bash index a96276af2..821b162ee 100644 --- a/jenkins-scripts/lib/_homebrew_github_setup.bash +++ b/jenkins-scripts/lib/_homebrew_github_setup.bash @@ -8,20 +8,11 @@ echo '# BEGIN SECTION: check variables' if [ -z "${PULL_REQUEST_HEAD_REPO}" ]; then echo PULL_REQUEST_HEAD_REPO not specified, setting to osrfbuild echo - PULL_REQUEST_HEAD_REPO=git@github.com:osrfbuild/homebrew-simulation.git + # personal tokens only support https + PULL_REQUEST_HEAD_REPO=https://github.com/osrfbuild/homebrew-simulation.git fi echo '# END SECTION' -echo '# BEGIN SECTION: check github perms' -# Github autentication. git access is provided by public key access -# and hub cli needs a token -if [[ -z $(ssh -T git@github.com 2>&1 | grep successfully) ]]; then - echo "The github connection seems not to be valid:" - ssh -T git@github.com - echo "Please check that the ssh key authentication is working" - exit 1 -fi - echo '# BEGIN SECTION: download linuxbrew' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" echo '# END SECTION' diff --git a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash index a23c8d512..7ac16f93e 100644 --- a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash +++ b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash @@ -26,7 +26,7 @@ if [ -z "${SOURCE_TARBALL_SHA}" ]; then fi echo '# END SECTION' -PULL_REQUEST_HEAD_REPO=git@github.com:osrfbuild/homebrew-simulation.git +PULL_REQUEST_HEAD_REPO=https://github.com/osrfbuild/homebrew-simulation.git . ${SCRIPT_LIBDIR}/_homebrew_github_setup.bash . ${SCRIPT_LIBDIR}/_homebrew_github_get_formula_path.bash From 2f83bee5e3e545562245686af17732125f93f84f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 26 Feb 2021 18:08:08 +0100 Subject: [PATCH 17/21] Use GITHUB_TOKEN in git push --- jenkins-scripts/dsl/test.dsl | 15 +++++++++++++++ jenkins-scripts/lib/_homebrew_github_commit.bash | 2 +- jenkins-scripts/lib/_homebrew_github_setup.bash | 8 ++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index d7bc3d2a5..42fa1ef0d 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -46,6 +46,21 @@ test_credentials_token_job.with """.stripIndent()) } + postBuildScripts { + steps { + shell("""\ + #!/bin/bash -xe + + # remove token after the build ends unconditionally + rm -fr \${WORKSPACE}/homebrew-simulation/.git/config + """.stripIndent()) + } + + onlyIfBuildSucceeds(false) + onlyIfBuildFails(false) + } + + wrappers { preBuildCleanup() } diff --git a/jenkins-scripts/lib/_homebrew_github_commit.bash b/jenkins-scripts/lib/_homebrew_github_commit.bash index e6dda3318..ca1cf691c 100644 --- a/jenkins-scripts/lib/_homebrew_github_commit.bash +++ b/jenkins-scripts/lib/_homebrew_github_commit.bash @@ -61,7 +61,7 @@ ${GIT} status echo ${GIT} show HEAD echo -${GIT} push -u pr_head ${PULL_REQUEST_BRANCH} +${GIT} push -u pr_head ${PULL_REQUEST_BRANCH} > ${WORKSPACE}/git-push.log # Create a pull request if one doesn't yet exist diff --git a/jenkins-scripts/lib/_homebrew_github_setup.bash b/jenkins-scripts/lib/_homebrew_github_setup.bash index 821b162ee..0517f27e0 100644 --- a/jenkins-scripts/lib/_homebrew_github_setup.bash +++ b/jenkins-scripts/lib/_homebrew_github_setup.bash @@ -29,6 +29,14 @@ ${BREW} tap osrf/simulation TAP_PREFIX=$(${BREW} --repo osrf/simulation) GIT="git -C ${TAP_PREFIX}" ${GIT} remote add pr_head ${PULL_REQUEST_HEAD_REPO} +# manage credentials +if [[ -z ${GITHUB_TOKEN} ]]; then + echo "No GITHUB_TOKEN defined. Check the DSL configuration" + exit 1 +fi +set +x +git config url."https://osrfbuild:\${GITHUB_TOKEN}@github.com/osrfbuild/homebrew-simulation.git".InsteadOf https://github.com/osrfbuild/homebrew-simulation.git +set -x # unshallow to get a full clone able to push ${GIT} fetch --unshallow || true ${GIT} fetch pr_head From 0693a381be2cd915245878d2ec25299ea0f60810 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 26 Feb 2021 18:20:35 +0100 Subject: [PATCH 18/21] Fix DSL --- jenkins-scripts/dsl/test.dsl | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index 42fa1ef0d..d62bbe383 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -46,21 +46,23 @@ test_credentials_token_job.with """.stripIndent()) } - postBuildScripts { - steps { - shell("""\ - #!/bin/bash -xe + publishers + { + postBuildScripts { + steps { + shell("""\ + #!/bin/bash -xe - # remove token after the build ends unconditionally - rm -fr \${WORKSPACE}/homebrew-simulation/.git/config - """.stripIndent()) - } + # remove token after the build ends unconditionally + rm -fr \${WORKSPACE}/homebrew-simulation/.git/config + """.stripIndent()) + } - onlyIfBuildSucceeds(false) - onlyIfBuildFails(false) + onlyIfBuildSucceeds(false) + onlyIfBuildFails(false) + } } - wrappers { preBuildCleanup() } From 79ac87c8f455ece8d2cf10a74630b852b32f36a7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 3 Jun 2021 20:46:47 +0200 Subject: [PATCH 19/21] Run job on osx nodes --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 52c9a2d53..e1d9386ea 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -60,7 +60,7 @@ release_job.with String PR_URL_export_file_name = 'pull_request_created.properties' String PR_URL_export_file = '${WORKSPACE}/' + PR_URL_export_file_name - label "master" + label "osx" wrappers { preBuildCleanup() From 25f20128dff624861cc963758d4935eadedd7ccb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 11 Oct 2021 23:33:15 +0200 Subject: [PATCH 20/21] Do not need to run jobs on master --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index e1d9386ea..744cd9caf 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -243,7 +243,7 @@ GitHubCredentials.createOsrfbuildToken(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with { - label "master" + label "osx" wrappers { From 65a596759209eca360dff577626ffc2e0b98132b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 7 Dec 2023 19:32:57 +0100 Subject: [PATCH 21/21] Back to run jobs on master Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/brew_release.dsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 76fa853b7..e6ea081a1 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -60,7 +60,7 @@ release_job.with String PR_URL_export_file_name = 'pull_request_created.properties' String PR_URL_export_file = '${WORKSPACE}/' + PR_URL_export_file_name - label "osx" + label Globals.nontest_label("master") wrappers { preBuildCleanup() @@ -244,7 +244,7 @@ GitHubCredentials.createOsrfbuildToken(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with { - label "osx" + label Globals.nontest_label("master") wrappers {