From 9d4369c642e723564dd74d240407269b2b75f6bf Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 13:24:28 +0100 Subject: [PATCH 1/8] mockbuild: remove unused logs These logs are not preserved, just remove them. --- schutzbot/mockbuild.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index 7bbbb94..1a0cb32 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -74,9 +74,8 @@ sudo mock -v -r $MOCK_CONFIG --resultdir repo/$REPO_DIR --with=tests \ # Change the ownership of all of our repo files from root to our CI user. sudo chown -R $USER repo/${REPO_DIR%%/*} -# Move the logs out of the way. -greenprint "🧹 Retaining logs from mock build" -mv repo/${REPO_DIR}/*.log $WORKSPACE +greenprint " Remove logs from mock build" +rm repo/${REPO_DIR}/*.log # Create a repo of the built RPMs. greenprint "⛓️ Creating dnf repository" From 4496fe91f9450394956f4267a7ff60b0ac5ce343 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 13:24:51 +0100 Subject: [PATCH 2/8] mockbuild: remove unused variable --- schutzbot/mockbuild.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index 1a0cb32..2ac97ec 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -35,9 +35,6 @@ if ! s3cmd --version > /dev/null 2>&1; then sudo pip3 -q install s3cmd fi -# Jenkins sets a workspace variable as the root of its working directory. -WORKSPACE=${WORKSPACE:-$(pwd)} - # Mock configuration file to use for building RPMs. MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)" From 6ccc0f068e8283119aae2818754c95106c406f6a Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 13:51:24 +0100 Subject: [PATCH 3/8] mockbuild: change repository path Change the repository path on S3 to a more predictable one, mirroring the pattern we're using for osbuild-composer. Notably, don't use short commit ids. The length of these is not predictable. It depends on the shortest unique prefix in the repository and git configuration. For example, koji-osbuild commit $SHA for fedora-33 on x86_64 will result in this URL: koji-osbuild/fedora-33/x86_64/$SHA --- schutzbot/mockbuild.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index 2ac97ec..b08b025 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -39,7 +39,7 @@ fi MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)" # The commit we are testing -GIT_SHA=$(git rev-parse --short HEAD) +GIT_SHA=$(git rev-parse HEAD) # Bucket in S3 where our artifacts are uploaded REPO_BUCKET=osbuild-composer-repos @@ -47,11 +47,15 @@ REPO_BUCKET=osbuild-composer-repos # Public URL for the S3 bucket with our artifacts. MOCK_REPO_BASE_URL="http://${REPO_BUCKET}.s3-website.us-east-2.amazonaws.com" +# Relative path of the repository – used for constructing both the local and +# remote paths below, so that they're consistent. +REPO_PATH=koji-osbuild/${ID}-${VERSION_ID}/${ARCH}/${GIT_SHA} + # Directory to hold the RPMs temporarily before we upload them. -REPO_DIR=koji-osbuild/${GIT_SHA}/${ID}${VERSION_ID//./}_${ARCH} +REPO_DIR=repo/${REPO_PATH} # Full URL to the RPM repository after they are uploaded. -REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_DIR} +REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_PATH} # Print some data. greenprint "🧬 Using mock config: ${MOCK_CONFIG}" @@ -65,18 +69,18 @@ ninja -C build srpms # Compile RPMs in a mock chroot greenprint "🎁 Building RPMs with mock" -sudo mock -v -r $MOCK_CONFIG --resultdir repo/$REPO_DIR --with=tests \ +sudo mock -v -r $MOCK_CONFIG --resultdir $REPO_DIR --with=tests \ build/rpmbuild/SRPMS/*.src.rpm # Change the ownership of all of our repo files from root to our CI user. -sudo chown -R $USER repo/${REPO_DIR%%/*} +sudo chown -R $USER ${REPO_DIR%%/*} greenprint " Remove logs from mock build" -rm repo/${REPO_DIR}/*.log +rm ${REPO_DIR}/*.log # Create a repo of the built RPMs. greenprint "⛓️ Creating dnf repository" -createrepo_c repo/${REPO_DIR} +createrepo_c ${REPO_DIR} # Upload repository to S3. greenprint "☁ Uploading RPMs to S3" From 303b8beeb8ec2e4eff475f419a294d6161ac6607 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 17:44:34 +0100 Subject: [PATCH 4/8] schutzbot: don't copy dnf repo file between stages Repository URLs are predictable. There's no need to use Jenkins' stash feature to pass the repo file between stages. Instead, simply create the repo file where it is needed, in deploy.sh. --- schutzbot/Jenkinsfile | 10 ---------- schutzbot/deploy.sh | 13 +++++++++---- schutzbot/mockbuild.sh | 12 ------------ 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile index 3e897f1..14ac1f5 100644 --- a/schutzbot/Jenkinsfile +++ b/schutzbot/Jenkinsfile @@ -50,10 +50,6 @@ pipeline { retry(3) { sh "schutzbot/mockbuild.sh" } - stash ( - includes: 'mock.repo', - name: 'fedora32' - ) } } stage('EL8') { @@ -67,10 +63,6 @@ pipeline { retry(3) { sh "schutzbot/mockbuild.sh" } - stash ( - includes: 'mock.repo', - name: 'rhel8cdn' - ) } } } @@ -86,7 +78,6 @@ pipeline { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { - unstash 'fedora32' run_tests('integration') } post { @@ -103,7 +94,6 @@ pipeline { RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') } steps { - unstash 'rhel8cdn' run_tests('integration') } post { diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh index fb760ec..e2eb359 100755 --- a/schutzbot/deploy.sh +++ b/schutzbot/deploy.sh @@ -62,13 +62,18 @@ fi # Add osbuild team ssh keys. cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null -# Set up a dnf repository for the RPMs we built via mock. -sudo cp mock.repo /etc/yum.repos.d/ - # Set up dnf repositories with the RPMs we want to test sudo tee /etc/yum.repos.d/osbuild.repo << EOF -[osbuild] +[koji-osbuild] name=osbuild ${GIT_COMMIT} +baseurl=${DNF_REPO_BASEURL}/koji-osbuild/${ID}-${VERSION_ID}/${ARCH}/${GIT_COMMIT} +enabled=1 +gpgcheck=0 +# Default dnf repo priority is 99. Lower number means higher priority. +priority=5 + +[osbuild] +name=osbuild ${OSBUILD_COMMIT} baseurl=${DNF_REPO_BASEURL}/osbuild/${ID}-${VERSION_ID}/${ARCH}/${OSBUILD_COMMIT} enabled=1 gpgcheck=0 diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index b08b025..b6a07c5 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -87,15 +87,3 @@ greenprint "☁ Uploading RPMs to S3" pushd repo s3cmd --acl-public sync . s3://${REPO_BUCKET}/ popd - -# Create a repository file. -greenprint "πŸ“œ Generating dnf repository file" -tee mock.repo << EOF -[schutzbot-mock] -name=schutzbot mock ${GIT_SHA} ${ID}${VERSION_ID//./} -baseurl=${REPO_URL} -enabled=1 -gpgcheck=0 -# Default dnf repo priority is 99. Lower number means higher priority. -priority=5 -EOF From d8f287d86b45ce2541a4b02e4e153214320259ce Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 17:45:28 +0100 Subject: [PATCH 5/8] mockbuild: don't build repository when it already exists This avoids doing unnecessary work and speeds up restarting tests. --- schutzbot/mockbuild.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index b6a07c5..c670e03 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -57,6 +57,12 @@ REPO_DIR=repo/${REPO_PATH} # Full URL to the RPM repository after they are uploaded. REPO_URL=${MOCK_REPO_BASE_URL}/${REPO_PATH} +# Don't rerun the build if it already exists +if curl --silent --fail --head --output /dev/null "${REPO_URL}/repodata/repomd.xml"; then + greenprint "🎁 Repository already exists. Exiting." + exit 0 +fi + # Print some data. greenprint "🧬 Using mock config: ${MOCK_CONFIG}" greenprint "πŸ“¦ Git SHA: ${GIT_SHA}" From 494a2abf914fef3b29244357234312cd87b03f67 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 17:47:06 +0100 Subject: [PATCH 6/8] mockbuild: move installing dependencies down These don't need to run when we're not building anything. --- schutzbot/mockbuild.sh | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index c670e03..4f51253 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -10,31 +10,6 @@ function greenprint { source /etc/os-release ARCH=$(uname -m) -# Mock is only available in EPEL for RHEL. -if [[ $ID == rhel ]] && ! rpm -q epel-release; then - greenprint "πŸ“¦ Setting up EPEL repository" - curl -Ls --retry 5 --output /tmp/epel.rpm \ - https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - sudo rpm -Uvh /tmp/epel.rpm -fi - -# Register RHEL if we are provided with a registration script. -if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then - greenprint "πŸͺ™ Registering RHEL instance" - sudo chmod +x $RHN_REGISTRATION_SCRIPT - sudo $RHN_REGISTRATION_SCRIPT -fi - -# Install requirements for building RPMs in mock. -greenprint "πŸ“¦ Installing mock requirements" -sudo dnf -y install createrepo_c meson mock ninja-build python3-pip rpm-build - -# Install s3cmd if it is not present. -if ! s3cmd --version > /dev/null 2>&1; then - greenprint "πŸ“¦ Installing s3cmd" - sudo pip3 -q install s3cmd -fi - # Mock configuration file to use for building RPMs. MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)" @@ -63,6 +38,31 @@ if curl --silent --fail --head --output /dev/null "${REPO_URL}/repodata/repomd.x exit 0 fi +# Mock is only available in EPEL for RHEL. +if [[ $ID == rhel ]] && ! rpm -q epel-release; then + greenprint "πŸ“¦ Setting up EPEL repository" + curl -Ls --retry 5 --output /tmp/epel.rpm \ + https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + sudo rpm -Uvh /tmp/epel.rpm +fi + +# Register RHEL if we are provided with a registration script. +if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then + greenprint "πŸͺ™ Registering RHEL instance" + sudo chmod +x $RHN_REGISTRATION_SCRIPT + sudo $RHN_REGISTRATION_SCRIPT +fi + +# Install requirements for building RPMs in mock. +greenprint "πŸ“¦ Installing mock requirements" +sudo dnf -y install createrepo_c meson mock ninja-build python3-pip rpm-build + +# Install s3cmd if it is not present. +if ! s3cmd --version > /dev/null 2>&1; then + greenprint "πŸ“¦ Installing s3cmd" + sudo pip3 -q install s3cmd +fi + # Print some data. greenprint "🧬 Using mock config: ${MOCK_CONFIG}" greenprint "πŸ“¦ Git SHA: ${GIT_SHA}" From ed424cd5923b38f8755946c181ebea48725e682a Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 18:08:16 +0100 Subject: [PATCH 7/8] koji-osbuild.spec: don't generate from .in file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove koji-osbuild.spec.in and the bits in meson.build that generate the final spec file out of it. Pull in the changelog from how OndΕ™ej Budai does it in osbuild-composer, because it's adorable. When releasing, the version now has to be bumped in the spec file. This is to make it consistent with other osbuild projects, and to simplify reverse dependency testing. --- koji-osbuild.spec.in => koji-osbuild.spec | 10 +++--- meson.build | 41 ++++++----------------- 2 files changed, 16 insertions(+), 35 deletions(-) rename koji-osbuild.spec.in => koji-osbuild.spec (96%) diff --git a/koji-osbuild.spec.in b/koji-osbuild.spec similarity index 96% rename from koji-osbuild.spec.in rename to koji-osbuild.spec index f0b5730..8846678 100644 --- a/koji-osbuild.spec.in +++ b/koji-osbuild.spec @@ -1,8 +1,7 @@ -%global commit @commit@ %global forgeurl https://github.com/osbuild/koji-osbuild Name: koji-osbuild -Version: @version@ +Version: 3 Release: 0%{?dist} Summary: Koji integration for osbuild composer @@ -152,5 +151,8 @@ Integration tests for koji-osbuild. To be run on a dedicated system. %changelog -* @longdate@ osbuild - @version@-@reltag@ -- build from git sources. +# the changelog is distribution-specific, therefore there's just one entry +# to make rpmlint happy. + +* Tue Aug 25 2020 Image Builder team - 0-1 +- On this day, this project was born. diff --git a/meson.build b/meson.build index 96cb8a0..deb6867 100644 --- a/meson.build +++ b/meson.build @@ -1,4 +1,4 @@ -project('koji-osbuild', license: 'ASL 2.0', version: '3') +project('koji-osbuild', license: 'ASL 2.0') srcdir = meson.source_root() @@ -16,33 +16,12 @@ install_data( ) git = find_program('git') - -subs = configuration_data() -subs.set('version', meson.project_version()) - -longdate = run_command('date', '+%a %b %d %Y').stdout().strip() -subs.set('longdate', longdate) - -gitres = run_command(git, ['--git-dir=@0@/.git'.format(srcdir), - 'rev-parse', - '--short=7', - 'HEAD']) - -if gitres.returncode() == 0 - commit = gitres.stdout().strip() - subs.set('commit', commit) - - gitrev = gitres.stdout().strip() - gitdate = run_command('date', '+%Y%m%d').stdout().strip() - # YYYYMMDD - subs.set('reltag', '@0@git@1@'.format(gitdate, gitrev)) -endif - -spec_file = configure_file( - input: 'koji-osbuild.spec.in', - output: 'koji-osbuild.spec', - configuration: subs +gitres = run_command( + git, + ['--git-dir=@0@/.git'.format(srcdir), 'rev-parse', 'HEAD'], + check=True ) +commit = gitres.stdout().strip() archive_name = meson.project_name() + '-' + gitrev full_archive_name = archive_name + '.tar.gz' @@ -79,7 +58,8 @@ srpm_target = custom_target( rpmbuild, '-bs', spec_file, '--define', '_sourcedir ' + meson.build_root(), - '--define', '_topdir ' + rpmdirs_target.full_path() + '--define', '_topdir ' + rpmdirs_target.full_path(), + '--define', 'commit ' + commit ], output: 'srpms', depends: [archive] @@ -91,15 +71,14 @@ rpm_target = custom_target( rpmbuild, '-ba', spec_file, '--define', '_sourcedir ' + meson.build_root(), - '--define', '_topdir ' + rpmdirs_target.full_path() + '--define', '_topdir ' + rpmdirs_target.full_path(), + '--define', 'commit ' + commit ], output: 'rpms', depends: [archive] ) msg = ['', - 'version: @0@'.format(meson.project_version()), - '', 'hub plugins path: @0@'.format(hub_plugins_path), 'builder plugins path: @0@'.format(builder_plugins_path), '', From 160b755e6ac9f8bb9a01f20028d875d05e1ffc64 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Fri, 20 Nov 2020 18:46:30 +0100 Subject: [PATCH 8/8] mockbuild: use mock to build source rpm Running `meson build` on the host can have different results from building the SRPM in mock. --- schutzbot/mockbuild.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index 4f51253..6727c51 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -55,7 +55,7 @@ fi # Install requirements for building RPMs in mock. greenprint "πŸ“¦ Installing mock requirements" -sudo dnf -y install createrepo_c meson mock ninja-build python3-pip rpm-build +sudo dnf -y install createrepo_c mock python3-pip rpm-build # Install s3cmd if it is not present. if ! s3cmd --version > /dev/null 2>&1; then @@ -68,15 +68,19 @@ greenprint "🧬 Using mock config: ${MOCK_CONFIG}" greenprint "πŸ“¦ Git SHA: ${GIT_SHA}" greenprint "πŸ“€ RPMS will be uploaded to: ${REPO_URL}" -# Build source RPMs. -greenprint "πŸ”§ Building source RPMs." -meson build -ninja -C build srpms - -# Compile RPMs in a mock chroot -greenprint "🎁 Building RPMs with mock" -sudo mock -v -r $MOCK_CONFIG --resultdir $REPO_DIR --with=tests \ - build/rpmbuild/SRPMS/*.src.rpm +greenprint "πŸ”§ Building source RPM" +git archive --prefix "koji-osbuild-${GIT_SHA}/" --output "koji-osbuild-${GIT_SHA}.tar.gz" HEAD +sudo mock -v -r "$MOCK_CONFIG" --buildsrpm \ + --define "commit ${GIT_SHA}" \ + --spec ./koji-osbuild.spec \ + --sources "./koji-osbuild-${GIT_SHA}.tar.gz" \ + --resultdir ./srpm + +greenprint "🎁 Building RPMs" +sudo mock -v -r $MOCK_CONFIG \ + --define "commit ${GIT_SHA}" \ + --resultdir $REPO_DIR \ + srpm/*.src.rpm # Change the ownership of all of our repo files from root to our CI user. sudo chown -R $USER ${REPO_DIR%%/*}