From 64606e15313e92e6c8ef00f767406346d87bd99e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 19 Jan 2021 18:44:05 +0000 Subject: [PATCH] Add stream metadata for RHCOS (bump x86_64 to 48.83.202102230316-0) This implements part of the plan from: https://github.com/openshift/os/issues/477 When we originally added the pinned RHCOS metadata `rhcos.json` to the installer, we also changed the coreos-assembler `meta.json` format into an arbitrary new format in the name of some cleanups. In retrospect, this was a big mistake because we now have two formats. Then Fedora CoreOS appeared and added streams JSON as a public API. We decided to unify on streams metadata; there's now a published Go library for it: https://github.com/coreos/stream-metadata-go Among other benefits, it is a single file that supports multiple architectures. UPI installs should now use stream metadata, particularly to find public cloud images. This is exposed via a new `openshift-install coreos print-stream-json` command. This is an important preparatory step for exposing this via `oc` as well as having something in the cluster update to it. HOWEVER as a (really hopefully temporary) hack, we *duplicate* the metadata so that IPI installs use the new stream format, and UPI CI jobs can still use the old format (with different RHCOS versions). We will port the UPI docs and CI jobs after this merges. --- cmd/openshift-install/coreos.go | 11 + cmd/openshift-install/main.go | 1 + data/data/rhcos-4.8.json | 394 +++++++++++++++++++++++++ docs/user/aws/install_upi.md | 4 +- docs/user/metal/customization_ipi.md | 6 +- docs/user/openstack/customization.md | 4 +- docs/user/overview.md | 31 ++ hack/update-rhcos-bootimage.py | 19 ++ pkg/asset/cluster/tfvars.go | 21 +- pkg/asset/manifests/coreosbootimage.go | 99 +++++++ pkg/asset/manifests/operators.go | 5 +- pkg/asset/rhcos/bootstrap_image.go | 45 ++- pkg/asset/rhcos/image.go | 94 +++--- pkg/coreoscli/cmd.go | 40 +++ pkg/rhcos/ami.go | 26 -- pkg/rhcos/azure.go | 24 -- pkg/rhcos/builds.go | 111 ++++--- pkg/rhcos/gcp.go | 30 -- pkg/rhcos/openstack.go | 39 --- pkg/rhcos/qemu.go | 43 --- pkg/rhcos/vmware.go | 43 --- 21 files changed, 780 insertions(+), 310 deletions(-) create mode 100644 cmd/openshift-install/coreos.go create mode 100644 data/data/rhcos-4.8.json create mode 100644 pkg/asset/manifests/coreosbootimage.go create mode 100644 pkg/coreoscli/cmd.go delete mode 100644 pkg/rhcos/ami.go delete mode 100644 pkg/rhcos/azure.go delete mode 100644 pkg/rhcos/gcp.go delete mode 100644 pkg/rhcos/qemu.go delete mode 100644 pkg/rhcos/vmware.go diff --git a/cmd/openshift-install/coreos.go b/cmd/openshift-install/coreos.go new file mode 100644 index 00000000000..61aa99fe2ba --- /dev/null +++ b/cmd/openshift-install/coreos.go @@ -0,0 +1,11 @@ +package main + +import ( + "github.com/spf13/cobra" + + "github.com/openshift/installer/pkg/coreoscli" +) + +func newCoreOSCmd() *cobra.Command { + return coreoscli.NewCmd() +} diff --git a/cmd/openshift-install/main.go b/cmd/openshift-install/main.go index 48bf7b6ebf0..a7699b6d5f5 100644 --- a/cmd/openshift-install/main.go +++ b/cmd/openshift-install/main.go @@ -60,6 +60,7 @@ func installerMain() { newGatherCmd(), newVersionCmd(), newGraphCmd(), + newCoreOSCmd(), newCompletionCmd(), newMigrateCmd(), newExplainCmd(), diff --git a/data/data/rhcos-4.8.json b/data/data/rhcos-4.8.json new file mode 100644 index 00000000000..840b90dd463 --- /dev/null +++ b/data/data/rhcos-4.8.json @@ -0,0 +1,394 @@ +{ + "stream": "rhcos-4.8", + "metadata": { + "last-modified": "2021-02-23T21:27:14Z" + }, + "architectures": { + "ppc64le": { + "artifacts": { + "metal": { + "release": "47.83.202102091015-0", + "formats": { + "4k.raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-metal4k.ppc64le.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-metal4k.ppc64le.raw.gz.sig", + "sha256": "f6e4458958f38a7bccae4a89a73134d431b86b183ebcf76542446c959fa2d520", + "uncompressed-sha256": "142ce5a0cb3984611a74d5d9d99ee6e320029802d2bee7c31d6b6b9455f3d293" + } + }, + "iso": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live.ppc64le.iso", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live.ppc64le.iso.sig", + "sha256": "d6fb6f949569461e3f9eb67883fa00b611275bb7d7bd3d9f11beaf1c0b33d389" + } + }, + "pxe": { + "kernel": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-kernel-ppc64le", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-kernel-ppc64le.sig", + "sha256": "e310166a16592a5cd1bc152b07fda84278b251dc385cf000c65ed7bb31d254ea" + }, + "initramfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-initramfs.ppc64le.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-initramfs.ppc64le.img.sig", + "sha256": "a15c4eaaf5aa0176fc475e4ec41df4ca0e83595f7a5e561e2de0a26f8a485b60" + }, + "rootfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-rootfs.ppc64le.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-live-rootfs.ppc64le.img.sig", + "sha256": "621ed54c0cb5180e7e798778c5c365f04b328aa983bc5a752ecea3a17ffd5bce" + } + }, + "raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-metal.ppc64le.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-metal.ppc64le.raw.gz.sig", + "sha256": "d0fa64d744dd731ad3185b39ada8d6eb886c8dbba3e683a30176838bdf20ef9a", + "uncompressed-sha256": "1880a77eed6741e45346102b85b5cceb071d7da5479b1075efd810767a33693c" + } + } + } + }, + "openstack": { + "release": "47.83.202102091015-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-openstack.ppc64le.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-openstack.ppc64le.qcow2.gz.sig", + "sha256": "47865a06d1ca6ae06481ab6f1d52ed6c9fca02ae539a06639c8bea65173a1550", + "uncompressed-sha256": "443ed7133f49e36138ac7c452bedcca2d8fde02f818a43fce1d905d41d0a8cea" + } + } + } + }, + "qemu": { + "release": "47.83.202102091015-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-qemu.ppc64le.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-ppc64le/47.83.202102091015-0/ppc64le/rhcos-47.83.202102091015-0-qemu.ppc64le.qcow2.gz.sig", + "sha256": "dce14413bb0347ffcb793a529ed2b4e39efa2b523a3e63faf552e574752a5aac", + "uncompressed-sha256": "2044dd7198fd1f730c3fe1d68c422be7e06ed57b4ad28bfe402f4836b84be869" + } + } + } + } + }, + "images": {} + }, + "s390x": { + "artifacts": { + "metal": { + "release": "47.83.202102090311-0", + "formats": { + "4k.raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-metal4k.s390x.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-metal4k.s390x.raw.gz.sig", + "sha256": "566bca2f1462b4d70f1d7da7a1c7d477252f2e2db6251ee61f6e810c3a450982", + "uncompressed-sha256": "3478e62963f4b0bba45cc41a92cae33e9907cc3e4ed1206ef94a05db4e70b9fe" + } + }, + "iso": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live.s390x.iso", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live.s390x.iso.sig", + "sha256": "e64d8d66bd69e6ea7ebdd9be74abce43e10e0731527ee6662780f20863ffc71c" + } + }, + "pxe": { + "kernel": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-kernel-s390x", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-kernel-s390x.sig", + "sha256": "a0f17299369b9ce9e42c874a8cc3658a00e58144fc0f54849ef96d9414d811d0" + }, + "initramfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-initramfs.s390x.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-initramfs.s390x.img.sig", + "sha256": "07e5642b818ee9582f59050657dceecad4683c6c54c2f92b071549c8a4826e05" + }, + "rootfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-rootfs.s390x.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-live-rootfs.s390x.img.sig", + "sha256": "79de4fb97a151b051201eb04f291a1b36f54d3968d2c3f31571066da8231945a" + } + }, + "raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-metal.s390x.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-metal.s390x.raw.gz.sig", + "sha256": "299bcb3de103f701866904f41e316e7c375fd1e0c4434b068e578a35d11084e1", + "uncompressed-sha256": "ab0b1f6080c472f88dc418610cb6af2dd9e8aff835c9a7b417716f67c3e2e25d" + } + } + } + }, + "openstack": { + "release": "47.83.202102090311-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-openstack.s390x.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-openstack.s390x.qcow2.gz.sig", + "sha256": "4ebac56f35e48ff66b18ed21785f722272bc5b24573c2410f3d63a374eeab6db", + "uncompressed-sha256": "2920cfe9a35e24302127507ab39e54d619c839d910361fe45c0c46dbd9813e7a" + } + } + } + }, + "qemu": { + "release": "47.83.202102090311-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-qemu.s390x.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.7-s390x/47.83.202102090311-0/s390x/rhcos-47.83.202102090311-0-qemu.s390x.qcow2.gz.sig", + "sha256": "80d61716f6ca97e9b5256b2202132dcac169700937b40545ce9cd8ad9e6bbcee", + "uncompressed-sha256": "cc3b8294320a6635bfdf05a380fb931875a57fe6b18118940b1ce2f23ca069a1" + } + } + } + } + }, + "images": {} + }, + "x86_64": { + "artifacts": { + "aws": { + "release": "48.83.202102230316-0", + "formats": { + "vmdk.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-aws.x86_64.vmdk.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-aws.x86_64.vmdk.gz.sig", + "sha256": "19a8ebccb96db35e7d072a189403f3bc702d562bfe7ac23dc0fa86bb66a8bbd1", + "uncompressed-sha256": "654df2c248f511534b56d87559bf20a102885c868d0294fbb54989ba451be9c0" + } + } + } + }, + "azure": { + "release": "48.83.202102230316-0", + "formats": { + "vhd.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-azure.x86_64.vhd.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-azure.x86_64.vhd.gz.sig", + "sha256": "ff778846596fe69f9f916240cec39b5382c0bfb1bb5d0c7d7d4a94ef18c657ff", + "uncompressed-sha256": "c9d7bb3d13cc9352514631d7a73d2960dbbb5d6cc6bcdc0506b09258cfee3123" + } + } + } + }, + "gcp": { + "release": "48.83.202102230316-0", + "formats": { + "tar.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-gcp.x86_64.tar.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-gcp.x86_64.tar.gz.sig", + "sha256": "e94c5a30c30b9ab1a7c29014e07ad27943ece4a90e486699d423f0bdb9ecf730" + } + } + } + }, + "ibmcloud": { + "release": "48.83.202102230316-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-ibmcloud.x86_64.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-ibmcloud.x86_64.qcow2.gz.sig", + "sha256": "7d37fde683cc8cd94b3b6c9abb598ba80bc9a6b3ee49ccce87d697ca654e2d31", + "uncompressed-sha256": "481f6b1398bf38968bd0b9641390adde813762060a383fe38ca43c7d8076d146" + } + } + } + }, + "metal": { + "release": "48.83.202102230316-0", + "formats": { + "4k.raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-metal4k.x86_64.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-metal4k.x86_64.raw.gz.sig", + "sha256": "b26b8ae89d2283395703e86cb10f9320fd917bf86d9c858b342fb3f4b3a3c4e8", + "uncompressed-sha256": "95e33871092da5f50458d9d2d866db59ae52542fd00286e37cb2bca85163c435" + } + }, + "iso": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live.x86_64.iso", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live.x86_64.iso.sig", + "sha256": "e06bdece79c3eac45514d272c38bf5aca55a756e6e2f9aa99bbffe0d534501bd" + } + }, + "pxe": { + "kernel": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-kernel-x86_64", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-kernel-x86_64.sig", + "sha256": "806623984883fee24f94bb2f5944d87bbc380c43bbf8ca1f40d5b0f1981af8f5" + }, + "initramfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-initramfs.x86_64.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-initramfs.x86_64.img.sig", + "sha256": "8c3cefede4a29a05cc52033827728ffddad7fe63475a2a932ab07eb2ccc4a5d3" + }, + "rootfs": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-rootfs.x86_64.img", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-live-rootfs.x86_64.img.sig", + "sha256": "f253cd6559418ace05ef90632bf2d760ea9044c0af5c34acbd495dc2823cd83d" + } + }, + "raw.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-metal.x86_64.raw.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-metal.x86_64.raw.gz.sig", + "sha256": "103bb455f22607d9728ae166140c5d4afe43a1adb842912a5d33623d3df182f9", + "uncompressed-sha256": "f03c75edf6d415f29cf7f1b9dab931bf4b695b28fbaea4fe0b04ecc0588b4ed5" + } + } + } + }, + "openstack": { + "release": "48.83.202102230316-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz.sig", + "sha256": "abc2add9746eb7be82e6919ec13aad8e9eae8cf073d8da6126d7c95ea0dee962", + "uncompressed-sha256": "9ed73a4e415ac670535c2188221e5a4a5f3e945bc2e03a65b1ed4fc76e5db6f2" + } + } + } + }, + "qemu": { + "release": "48.83.202102230316-0", + "formats": { + "qcow2.gz": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-qemu.x86_64.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-qemu.x86_64.qcow2.gz.sig", + "sha256": "fa92d674f73ed6ffae1178bd7f311e0e11a18f2d8bfb3cba8d25c9798f1e7cac", + "uncompressed-sha256": "fcc902d4aed0ecb0e3f5f0ba02cfc39646d32f4c75c1e94fc1ed7c3baa526377" + } + } + } + }, + "vmware": { + "release": "48.83.202102230316-0", + "formats": { + "ova": { + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-vmware.x86_64.ova", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-vmware.x86_64.ova.sig", + "sha256": "7f648fb87d29408b2273c998f2b51e26519ca3a664b613e8c6684932b0e644d6" + } + } + } + } + }, + "images": { + "aws": { + "regions": { + "af-south-1": { + "release": "48.83.202102230316-0", + "image": "ami-0d0b066652e7b2b6b" + }, + "ap-east-1": { + "release": "48.83.202102230316-0", + "image": "ami-0af11ac48bb289725" + }, + "ap-northeast-1": { + "release": "48.83.202102230316-0", + "image": "ami-0630c57d9304ac014" + }, + "ap-northeast-2": { + "release": "48.83.202102230316-0", + "image": "ami-0845d373ef74d96d2" + }, + "ap-south-1": { + "release": "48.83.202102230316-0", + "image": "ami-0265e31ba3349e741" + }, + "ap-southeast-1": { + "release": "48.83.202102230316-0", + "image": "ami-0f5c814382a24e0ee" + }, + "ap-southeast-2": { + "release": "48.83.202102230316-0", + "image": "ami-0b4a7ca38872b47fe" + }, + "ca-central-1": { + "release": "48.83.202102230316-0", + "image": "ami-02cdcf3a5eecb715d" + }, + "eu-central-1": { + "release": "48.83.202102230316-0", + "image": "ami-0416dc0707afccd4f" + }, + "eu-north-1": { + "release": "48.83.202102230316-0", + "image": "ami-01eecb0a825ec4742" + }, + "eu-south-1": { + "release": "48.83.202102230316-0", + "image": "ami-0ee83369622acc3f1" + }, + "eu-west-1": { + "release": "48.83.202102230316-0", + "image": "ami-0fb7dba75d65d4155" + }, + "eu-west-2": { + "release": "48.83.202102230316-0", + "image": "ami-058c7ac61333223aa" + }, + "eu-west-3": { + "release": "48.83.202102230316-0", + "image": "ami-0659ca7763ea177a9" + }, + "me-south-1": { + "release": "48.83.202102230316-0", + "image": "ami-01fd244c585409d77" + }, + "sa-east-1": { + "release": "48.83.202102230316-0", + "image": "ami-0c010b0cba5089fa5" + }, + "us-east-1": { + "release": "48.83.202102230316-0", + "image": "ami-00434fdc5b33448cc" + }, + "us-east-2": { + "release": "48.83.202102230316-0", + "image": "ami-065724ff75d9eec9c" + }, + "us-west-1": { + "release": "48.83.202102230316-0", + "image": "ami-0c548bdf93b74cd59" + }, + "us-west-2": { + "release": "48.83.202102230316-0", + "image": "ami-06b386b428d7fa2f1" + } + } + }, + "gcp": { + "project": "rhcos-cloud", + "name": "rhcos-48-83-202102230316-0-gcp-x86-64" + } + }, + "rhel-coreos-extensions": { + "azure-disk": { + "release": "48.83.202102230316-0", + "url": "https://rhcos.blob.core.windows.net/imagebucket/rhcos-48.83.202102230316-0-azure.x86_64.vhd" + } + } + } + } +} diff --git a/docs/user/aws/install_upi.md b/docs/user/aws/install_upi.md index 90c88b6b65e..ed7546f0538 100644 --- a/docs/user/aws/install_upi.md +++ b/docs/user/aws/install_upi.md @@ -21,8 +21,8 @@ $ openshift-install create install-config ### Optional: Create Encrypted AMIs The IPI-based installer creates an encrypted AMI by default. If you wish to have an encrypted AMI for UPI-based -installs, you will need to create it directly. You can find a list of the appropriate base AMIs -[here](../../../data/data/rhcos.json). +installs, you will need to create it directly. You can find a list of the appropriate base AMIs +as part of the CoreOS stream metadata [here](../../../data/data/rhcos-4.8.json). You will make an encrypted copy of the AMI according to the [AWS documentation][encrypted-copy]. diff --git a/docs/user/metal/customization_ipi.md b/docs/user/metal/customization_ipi.md index 6796c6f1bb1..cf785a1a28e 100644 --- a/docs/user/metal/customization_ipi.md +++ b/docs/user/metal/customization_ipi.md @@ -24,12 +24,12 @@ When doing a disconnected installation, the baremetal platform has the additional requirement that we have locations to download the RHCOS -images. The installer downloads these from a location described in -[/data/data/rhcos.json](/data/data/rhcos.json), but they can be +images. The installer downloads these from a CoreOS stream metadata +embedded in the installer code, but they can be overridden to point to a local mirror. The SHA256 parameter in the URLs are required, and should match the -uncompressed SHA256 from rhcos.json. +uncompressed SHA256 from the embedded file e.g. `rhcos-4.8.json`. * `bootstrapOSImage` (optional string): Override the image used for the diff --git a/docs/user/openstack/customization.md b/docs/user/openstack/customization.md index 0015005c0ff..7c4b47f2a79 100644 --- a/docs/user/openstack/customization.md +++ b/docs/user/openstack/customization.md @@ -112,13 +112,13 @@ sshKey: ssh-ed25519 AAAA... ## Image Overrides -Normally the installer downloads the RHCOS image from a predetermined location described in [data/data/rhcos.json](/data/data/rhcos.json)). But the download URL can be overridden, notably for disconnected installations. +The OpenShift installer currently pins the version of RHEL CoreOS. Since OpenShift 4.8, this is done via a "stream metadata" file embedded at [data/data/rhcos-4.8.json](/data/data/rhcos-4.8.json). Normally the installer downloads the RHCOS image from that location. But the download URL can be overridden, notably for disconnected installations. To do so and upload binary data from a custom location the user may set `clusterOSImage` parameter in the install config that points to that location, and then start the installation. In all other respects the process will be consistent with the default. **NOTE:** For this to work, the parameter value must be a valid http(s) URL. -**NOTE:** The optional `sha256` query parameter can be attached to the URL, which will force the installer to check the image file checksum before uploading it into Glance. +**NOTE:** The optional `sha256` query parameter can be attached to the URL, which will force the installer to check the uncompressed image file checksum before uploading it into Glance. Example: diff --git a/docs/user/overview.md b/docs/user/overview.md index 521069117a1..14ccba68009 100644 --- a/docs/user/overview.md +++ b/docs/user/overview.md @@ -84,3 +84,34 @@ As the unstable warning suggests, the presence of `manifests` and the names and It is occasionally useful to make alterations like this as one-off changes, but don't expect them to work on subsequent installer releases. [cluster-version]: https://github.com/openshift/cluster-version-operator/blob/master/docs/dev/clusterversion.md + +### CoreOS bootimages + +The `openshift-install` binary contains pinned versions of RHEL CoreOS "bootimages" (i.e. OpenStack `qcow2`, AWS AMI, bare metal `.iso` etc.). +Fully automated installs use these by default. + +For UPI (User Provisioned Infrastructure) installs, you can use the `openshift-install coreos print-stream-json` command to access information +about the bootimages in [CoreOS Stream Metadata](https://github.com/coreos/stream-metadata-go) format. + +For example, this command will print the `x86_64` AMI for `us-west-1`: + +``` +$ openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.images.aws.regions["us-west-1"].image' +ami-0c548bdf93b74cd59 +$ +``` + +For on-premise clouds (e.g. OpenStack) with UPI installs, you may need to manually copy +a bootimage into the infrastructure. Here's an example command to print the `x86_64` `qcow2` file for `openstack`: + +``` +$ openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.openstack.formats["qcow2.gz"]' +{ + "disk": { + "location": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz", + "signature": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.8/48.83.202102230316-0/x86_64/rhcos-48.83.202102230316-0-openstack.x86_64.qcow2.gz.sig", + "sha256": "abc2add9746eb7be82e6919ec13aad8e9eae8cf073d8da6126d7c95ea0dee962", + "uncompressed-sha256": "9ed73a4e415ac670535c2188221e5a4a5f3e945bc2e03a65b1ed4fc76e5db6f2" + } +} +``` diff --git a/hack/update-rhcos-bootimage.py b/hack/update-rhcos-bootimage.py index ba2ff1af0a2..9be44086c35 100755 --- a/hack/update-rhcos-bootimage.py +++ b/hack/update-rhcos-bootimage.py @@ -1,4 +1,23 @@ #!/usr/bin/env python3 +# As of 4.8 we are aiming to switch to stream metadata: +# https://github.com/openshift/enhancements/pull/679 +# That transition hasn't yet fully completed; there are two copies of the +# RHCOS metadata: +# +# - data/data/rhcos-4.8.json (stream format, 4.8+) +# - data/data/rhcos-$arch.json (openshift/installer specific, 4.7 and below) +# +# See https://github.com/coreos/coreos-assembler/pull/2000 in particular. +# +# The initial file data/data/rhcos-4.8 was generated this way: +# +# $ plume cosa2stream --name rhcos-4.8 --distro rhcos x86_64=48.83.202102230316-0 s390x=47.83.202102090311-0 ppc64le=47.83.202102091015-0 > data/data/rhcos-4.8.json +# +# To update the bootimage for one or more architectures, use e.g. +# +# $ plume cosa2stream --target data/data/rhcos-4.8.json --distro rhcos x86_64=48.83.202102230316-0 s390x=47.83.202102090311-0 ppc64le=47.83.202102091015-0 +# +# To update the legacy metadata, use: # Usage: ./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.6/46.82.202008260918-0/x86_64/meta.json amd64 import codecs,os,sys,json,argparse import urllib.parse diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index 2c4fee7dd41..41c5399c5fc 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -9,6 +9,7 @@ import ( "strings" igntypes "github.com/coreos/ignition/v2/config/v3_2/types" + coreosarch "github.com/coreos/stream-metadata-go/arch" gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1" kubevirtprovider "github.com/openshift/cluster-api-provider-kubevirt/pkg/apis/kubevirtprovider/v1alpha1" kubevirtutils "github.com/openshift/cluster-api-provider-kubevirt/pkg/utils" @@ -338,16 +339,30 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { } preexistingnetwork := installConfig.Config.GCP.Network != "" - imageRaw, err := rhcospkg.GCPRaw(ctx, installConfig.Config.ControlPlane.Architecture) + archName := coreosarch.RpmArch(string(installConfig.Config.ControlPlane.Architecture)) + st, err := rhcospkg.FetchCoreOSBuild(ctx) if err != nil { - return errors.Wrap(err, "failed to find Raw GCP image URL") + return err + } + streamArch, err := st.GetArchitecture(archName) + if err != nil { + return err + } + + img := streamArch.Images.Gcp + if img == nil { + return fmt.Errorf("%s: No GCP build found", st.FormatPrefix(archName)) } + // For backwards compatibility, we generate this URL to the image (only applies to RHCOS, not FCOS/OKD) + // right now. It will only be used if nested virt or other licenses are enabled, which we + // really should deprecate and remove - xref https://github.com/openshift/installer/pull/4696 + imageURL := fmt.Sprintf("https://storage.googleapis.com/rhcos/rhcos/%s.tar.gz", img.Name) data, err := gcptfvars.TFVars( gcptfvars.TFVarsSources{ Auth: auth, MasterConfigs: masterConfigs, WorkerConfigs: workerConfigs, - ImageURI: imageRaw, + ImageURI: imageURL, ImageLicenses: installConfig.Config.GCP.Licenses, PublicZoneName: publicZoneName, PublishStrategy: installConfig.Config.Publish, diff --git a/pkg/asset/manifests/coreosbootimage.go b/pkg/asset/manifests/coreosbootimage.go new file mode 100644 index 00000000000..03bead44d60 --- /dev/null +++ b/pkg/asset/manifests/coreosbootimage.go @@ -0,0 +1,99 @@ +package manifests + +import ( + "context" + "path/filepath" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/asset/installconfig" + "github.com/openshift/installer/pkg/rhcos" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // mcoNamespace is where we target the configmap. + mcoNamespace = "openshift-machine-config-operator" + // configmapName is the name of the config map + configmapName = "coreos-bootimages" + // streamKey is the key that contains the stream metadata + streamKey = "stream" +) + +var ( + coreOSBootimagesCfgFilename = filepath.Join(manifestDir, "machine-config-operator-bootimages-config.yml") +) + +// CoreOSBootimages generates the manifest with CoreOS stream metadata for the bootimages +type CoreOSBootimages struct { + ConfigMap *corev1.ConfigMap + File *asset.File +} + +var _ asset.WritableAsset = (*CoreOSBootimages)(nil) + +// Name returns a human friendly name for the asset. +func (*CoreOSBootimages) Name() string { + return "CoreOSBootimages Config" +} + +// Dependencies returns all of the dependencies directly needed to generate +// the asset. +func (*CoreOSBootimages) Dependencies() []asset.Asset { + return []asset.Asset{ + &installconfig.InstallConfig{}, + } +} + +// Generate generates the CoreOSBootimages config and its CRD. +func (s *CoreOSBootimages) Generate(dependencies asset.Parents) error { + installConfig := &installconfig.InstallConfig{} + dependencies.Get(installConfig) + + streamData, err := rhcos.FetchRawCoreOSStream(context.Background()) + if err != nil { + return err + } + + data := make(map[string]string) + data[streamKey] = string(streamData) + + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: mcoNamespace, + Name: configmapName, + }, + Data: data, + } + + cmData, err := yaml.Marshal(cm) + if err != nil { + return errors.Wrapf(err, "failed to create %s manifest", s.Name()) + } + s.ConfigMap = cm + s.File = &asset.File{ + Filename: coreOSBootimagesCfgFilename, + Data: cmData, + } + return nil +} + +// Files returns the files generated by the asset. +func (s *CoreOSBootimages) Files() []*asset.File { + if s.File != nil { + return []*asset.File{s.File} + } + return []*asset.File{} +} + +// Load returns false since this asset is not written to disk by the installer. +func (s *CoreOSBootimages) Load(f asset.FileFetcher) (bool, error) { + return false, nil +} diff --git a/pkg/asset/manifests/operators.go b/pkg/asset/manifests/operators.go index cc6b6c28b9f..d6b6e9482f0 100644 --- a/pkg/asset/manifests/operators.go +++ b/pkg/asset/manifests/operators.go @@ -61,6 +61,7 @@ func (m *Manifests) Dependencies() []asset.Asset { &Proxy{}, &Scheduler{}, &ImageContentSourcePolicy{}, + &CoreOSBootimages{}, &tls.RootCA{}, &tls.MCSCertKey{}, @@ -84,7 +85,8 @@ func (m *Manifests) Generate(dependencies asset.Parents) error { proxy := &Proxy{} scheduler := &Scheduler{} imageContentSourcePolicy := &ImageContentSourcePolicy{} - dependencies.Get(installConfig, ingress, dns, network, infra, proxy, scheduler, imageContentSourcePolicy) + bootimages := &CoreOSBootimages{} + dependencies.Get(installConfig, ingress, dns, network, infra, proxy, scheduler, imageContentSourcePolicy, bootimages) redactedConfig, err := redactedInstallConfig(*installConfig.Config) if err != nil { @@ -114,6 +116,7 @@ func (m *Manifests) Generate(dependencies asset.Parents) error { m.FileList = append(m.FileList, proxy.Files()...) m.FileList = append(m.FileList, scheduler.Files()...) m.FileList = append(m.FileList, imageContentSourcePolicy.Files()...) + m.FileList = append(m.FileList, bootimages.Files()...) asset.SortFiles(m.FileList) diff --git a/pkg/asset/rhcos/bootstrap_image.go b/pkg/asset/rhcos/bootstrap_image.go index 6fde584b5ba..ca750c47537 100644 --- a/pkg/asset/rhcos/bootstrap_image.go +++ b/pkg/asset/rhcos/bootstrap_image.go @@ -3,8 +3,11 @@ package rhcos import ( "context" + "fmt" "time" + "github.com/coreos/stream-metadata-go/arch" + "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/rhcos" @@ -37,28 +40,44 @@ func (i *BootstrapImage) Generate(p asset.Parents) error { p.Get(ic) config := ic.Config - var osimage string - var err error ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() + switch config.Platform.Name() { case baremetal.Name: - // Check for RHCOS image URL override - if boi := config.Platform.BareMetal.BootstrapOSImage; boi != "" { - osimage = boi - break + archName := arch.RpmArch(string(config.ControlPlane.Architecture)) + st, err := rhcos.FetchCoreOSBuild(ctx) + if err != nil { + return err + } + streamArch, err := st.GetArchitecture(archName) + if err != nil { + return err } + // Check for CoreOS image URL override + if boi := config.Platform.BareMetal.BootstrapOSImage; boi != "" { + *i = BootstrapImage(boi) + return nil + } // Baremetal IPI launches a local VM for the bootstrap node // Hence requires the QEMU image to use the libvirt backend - osimage, err = rhcos.QEMU(ctx, config.ControlPlane.Architecture) + if a, ok := streamArch.Artifacts["qemu"]; ok { + u, err := rhcos.FindArtifactURL(a) + if err != nil { + return err + } + *i = BootstrapImage(u) + return nil + } + return fmt.Errorf("%s: No qemu build found", st.FormatPrefix(archName)) default: // other platforms use the same image for all nodes - osimage, err = osImage(config) - } - if err != nil { - return err + u, err := osImage(config) + if err != nil { + return err + } + *i = BootstrapImage(u) + return nil } - *i = BootstrapImage(osimage) - return nil } diff --git a/pkg/asset/rhcos/image.go b/pkg/asset/rhcos/image.go index 12ca213c0ac..5f834acd4b3 100644 --- a/pkg/asset/rhcos/image.go +++ b/pkg/asset/rhcos/image.go @@ -7,7 +7,7 @@ import ( "os" "time" - "github.com/pkg/errors" + "github.com/coreos/stream-metadata-go/arch" "github.com/sirupsen/logrus" "github.com/openshift/installer/pkg/asset" @@ -66,67 +66,95 @@ func (i *Image) Generate(p asset.Parents) error { } func osImage(config *types.InstallConfig) (string, error) { - arch := config.ControlPlane.Architecture - - var osimage string - var err error ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) defer cancel() + + archName := arch.RpmArch(string(config.ControlPlane.Architecture)) + + st, err := rhcos.FetchCoreOSBuild(ctx) + if err != nil { + return "", err + } + streamArch, err := st.GetArchitecture(archName) + if err != nil { + return "", err + } switch config.Platform.Name() { case aws.Name: if len(config.Platform.AWS.AMIID) > 0 { - osimage = config.Platform.AWS.AMIID - break + return config.Platform.AWS.AMIID, nil } region := config.Platform.AWS.Region if !configaws.IsKnownRegion(config.Platform.AWS.Region) { region = "us-east-1" } - osimage, err = rhcos.AMI(ctx, arch, region) + osimage, err := st.GetAMI(archName, region) + if err != nil { + return "", err + } if region != config.Platform.AWS.Region { osimage = fmt.Sprintf("%s,%s", osimage, region) } + return osimage, nil case gcp.Name: - osimage, err = rhcos.GCP(ctx, arch) + if streamArch.Images.Gcp != nil { + img := streamArch.Images.Gcp + return fmt.Sprintf("projects/%s/global/images/%s", img.Project, img.Name), nil + } + return "", fmt.Errorf("%s: No GCP build found", st.FormatPrefix(archName)) case libvirt.Name: - osimage, err = rhcos.QEMU(ctx, arch) - case openstack.Name: - if oi := config.Platform.OpenStack.ClusterOSImage; oi != "" { - osimage = oi - break + // 𝅘𝅥𝅮 Everything's going to be a-ok 𝅘𝅥𝅮 + if a, ok := streamArch.Artifacts["qemu"]; ok { + return rhcos.FindArtifactURL(a) + } + return "", fmt.Errorf("%s: No qemu build found", st.FormatPrefix(archName)) + case ovirt.Name, kubevirt.Name, openstack.Name: + op := config.Platform.OpenStack + if op != nil { + if oi := op.ClusterOSImage; oi != "" { + return oi, nil + } + } + if a, ok := streamArch.Artifacts["openstack"]; ok { + return rhcos.FindArtifactURL(a) } - osimage, err = rhcos.OpenStack(ctx, arch) - case ovirt.Name: - osimage, err = rhcos.OpenStack(ctx, arch) - case kubevirt.Name: - osimage, err = rhcos.OpenStack(ctx, arch) + return "", fmt.Errorf("%s: No openstack build found", st.FormatPrefix(archName)) case azure.Name: - osimage, err = rhcos.VHD(ctx, arch) + ext := streamArch.RHELCoreOSExtensions + if ext == nil { + return "", fmt.Errorf("%s: No azure build found", st.FormatPrefix(archName)) + } + azd := ext.AzureDisk + if azd == nil { + return "", fmt.Errorf("%s: No azure build found", st.FormatPrefix(archName)) + } + return azd.URL, nil case baremetal.Name: - // Check for RHCOS image URL override + // Check for image URL override if oi := config.Platform.BareMetal.ClusterOSImage; oi != "" { - osimage = oi - break + return oi, nil } // Note that baremetal IPI currently uses the OpenStack image // because this contains the necessary ironic config drive // ignition support, which isn't enabled in the UPI BM images - osimage, err = rhcos.OpenStack(ctx, arch) + if a, ok := streamArch.Artifacts["openstack"]; ok { + return rhcos.FindArtifactURL(a) + } + return "", fmt.Errorf("%s: No openstack build found", st.FormatPrefix(archName)) case vsphere.Name: - // Check for RHCOS image URL override + // Check for image URL override if config.Platform.VSphere.ClusterOSImage != "" { - osimage = config.Platform.VSphere.ClusterOSImage - break + return config.Platform.VSphere.ClusterOSImage, nil } - osimage, err = rhcos.VMware(ctx, arch) + if a, ok := streamArch.Artifacts["vmware"]; ok { + return rhcos.FindArtifactURL(a) + } + return "", fmt.Errorf("%s: No vmware build found", st.FormatPrefix(archName)) case none.Name: + return "", nil default: - return "", errors.New("invalid Platform") - } - if err != nil { - return "", err + return "", fmt.Errorf("invalid platform %v", config.Platform.Name()) } - return osimage, nil } diff --git a/pkg/coreoscli/cmd.go b/pkg/coreoscli/cmd.go new file mode 100644 index 00000000000..ff597f4a727 --- /dev/null +++ b/pkg/coreoscli/cmd.go @@ -0,0 +1,40 @@ +package coreoscli + +import ( + "context" + "os" + + "github.com/openshift/installer/pkg/rhcos" + "github.com/spf13/cobra" +) + +func printStreamJSON(cmd *cobra.Command, _ []string) error { + streamData, err := rhcos.FetchRawCoreOSStream(context.Background()) + if err != nil { + return err + } + os.Stdout.Write(streamData) + return nil +} + +// NewCmd returns a subcommand for explain +func NewCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "coreos", + Short: "Commands for operating on CoreOS boot images", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + + printStreamCmd := &cobra.Command{ + Use: "print-stream-json", + Short: "Outputs the CoreOS stream metadata for the bootimages", + Args: cobra.ExactArgs(0), + RunE: printStreamJSON, + } + cmd.AddCommand(printStreamCmd) + + return cmd +} diff --git a/pkg/rhcos/ami.go b/pkg/rhcos/ami.go deleted file mode 100644 index 04cb15db911..00000000000 --- a/pkg/rhcos/ami.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:generate go run ami_regions_generate.go rhcos ../../data/data/rhcos-amd64.json ami_regions.go - -package rhcos - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" -) - -// AMI fetches the HVM AMI ID of the Red Hat Enterprise Linux CoreOS release. -func AMI(ctx context.Context, arch types.Architecture, region string) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - ami, ok := meta.AMIs[region] - if !ok { - return "", errors.Errorf("no RHCOS AMIs found in %s", region) - } - - return ami.HVM, nil -} diff --git a/pkg/rhcos/azure.go b/pkg/rhcos/azure.go deleted file mode 100644 index 14e8fc30bb2..00000000000 --- a/pkg/rhcos/azure.go +++ /dev/null @@ -1,24 +0,0 @@ -package rhcos - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" -) - -// VHD fetches the URL of the public Azure storage bucket containing the RHCOS image -func VHD(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - url := meta.Azure.URL - if url == "" { - return "", errors.New("no RHCOS Azure URL found") - } - - return url, nil -} diff --git a/pkg/rhcos/builds.go b/pkg/rhcos/builds.go index 76d0b64415f..ae30ecfe4ce 100644 --- a/pkg/rhcos/builds.go +++ b/pkg/rhcos/builds.go @@ -5,69 +5,84 @@ import ( "encoding/json" "fmt" "io/ioutil" - "os" + "net/url" + "github.com/coreos/stream-metadata-go/stream" "github.com/openshift/installer/data" "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" ) -var ( - errInvalidArch = fmt.Errorf("no build metadata for given architecture") -) +// pinnedRHCOSStream should match the version of OpenShift; it refers +// to a JSON file embedded in this repository. For more information, see +// https://github.com/openshift/os/issues/477 +// and https://github.com/openshift/enhancements/pull/201 +const pinnedRHCOSStream = "rhcos-4.8" -type metadata struct { - AMIs map[string]struct { - HVM string `json:"hvm"` - } `json:"amis"` - Azure struct { - Image string `json:"image"` - URL string `json:"url"` +// FetchRawCoreOSStream returns the raw stream metadata for the +// bootimages embedded in the installer. +func FetchRawCoreOSStream(ctx context.Context) ([]byte, error) { + filename := fmt.Sprintf("%s.json", pinnedRHCOSStream) + file, err := data.Assets.Open(filename) + if err != nil { + return nil, errors.Wrapf(err, "failed to read embedded CoreOS stream metadata %s", filename) } - GCP struct { - Image string `json:"image"` - Project string `json:"project"` - URL string `json:"url"` + defer file.Close() + + body, err := ioutil.ReadAll(file) + if err != nil { + return nil, errors.Wrap(err, "failed to read CoreOS stream metadata") } - BaseURI string `json:"baseURI"` - Images struct { - QEMU struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` - UncompressedSHA256 string `json:"uncompressed-sha256"` - } `json:"qemu"` - OpenStack struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` - UncompressedSHA256 string `json:"uncompressed-sha256"` - } `json:"openstack"` - VMware struct { - Path string `json:"path"` - SHA256 string `json:"sha256"` - } `json:"vmware"` - } `json:"images"` - OSTreeVersion string `json:"ostree-version"` + return body, nil } -func fetchRHCOSBuild(ctx context.Context, arch types.Architecture) (*metadata, error) { - file, err := data.Assets.Open(fmt.Sprintf("rhcos-%s.json", arch)) +// FetchCoreOSBuild returns the pinned version of RHEL/Fedora CoreOS used +// by the installer to provision the bootstrap node and control plane currently. +// For more information, see e.g. https://github.com/openshift/enhancements/pull/201 +func FetchCoreOSBuild(ctx context.Context) (*stream.Stream, error) { + body, err := FetchRawCoreOSStream(ctx) if err != nil { return nil, err } - defer file.Close() - - body, err := ioutil.ReadAll(file) - if os.IsNotExist(err) { - return nil, errInvalidArch - } else if err != nil { - return nil, err + var st stream.Stream + if err := json.Unmarshal(body, &st); err != nil { + return nil, errors.Wrap(err, "failed to parse CoreOS stream metadata") } + return &st, nil +} - var meta *metadata - if err := json.Unmarshal(body, &meta); err != nil { - return meta, errors.Wrap(err, "failed to parse RHCOS build metadata") +// FormatURLWithIntegrity squashes an artifact into a URL string +// with the uncompressed sha256 as a query parameter. This is necessary +// currently because various parts of the installer pass around this +// reference as a string, and it's also exposed to users via install-config overrides. +func FormatURLWithIntegrity(artifact *stream.Artifact) (string, error) { + u, err := url.Parse(artifact.Location) + if err != nil { + return "", fmt.Errorf("failed to parse artifact URL: %v", err) } + q := u.Query() + q.Set("sha256", artifact.UncompressedSha256) + u.RawQuery = q.Encode() + return u.String(), nil +} - return meta, nil +// FindArtifactURL uses the first recognized artifact type; this +// mainly abstracts over e.g. `qcow2.xz` and `qcow2.gz`. (FCOS uses +// xz, RHCOS uses gzip right now) +// Some platforms have multiple artifact types, e.g. `metal` has an ISO +// as well as PXE files, etc. Avoid using this for any +// use this for that platform. +func FindArtifactURL(artifacts stream.PlatformArtifacts) (string, error) { + var artifact *stream.Artifact + for _, v := range artifacts.Formats { + if v.Disk != nil { + if artifact != nil { + return "", fmt.Errorf("multiple \"disk\" artifacts found") + } + artifact = v.Disk + } + } + if artifact != nil { + return FormatURLWithIntegrity(artifact) + } + return "", fmt.Errorf("no \"disk\" artifact found") } diff --git a/pkg/rhcos/gcp.go b/pkg/rhcos/gcp.go deleted file mode 100644 index 96dbbf27833..00000000000 --- a/pkg/rhcos/gcp.go +++ /dev/null @@ -1,30 +0,0 @@ -package rhcos - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" -) - -// GCP fetches the URL of the public RHCOS image -func GCP(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - return fmt.Sprintf("projects/%s/global/images/%s", meta.GCP.Project, meta.GCP.Image), nil -} - -// GCPRaw fetches the URL of the public GCP storage bucket containing the RHCOS image -func GCPRaw(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - return meta.GCP.URL, nil -} diff --git a/pkg/rhcos/openstack.go b/pkg/rhcos/openstack.go index 7f25ac7d815..574ed125a9c 100644 --- a/pkg/rhcos/openstack.go +++ b/pkg/rhcos/openstack.go @@ -1,48 +1,9 @@ package rhcos import ( - "context" "net/url" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" ) -// OpenStack fetches the URL of the Red Hat Enterprise Linux CoreOS release, -// for the openstack platform -func OpenStack(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - base, err := url.Parse(meta.BaseURI) - if err != nil { - return "", err - } - - relOpenStack, err := url.Parse(meta.Images.OpenStack.Path) - if err != nil { - return "", err - } - - baseURL := base.ResolveReference(relOpenStack).String() - - // Attach sha256 checksum to the URL. Always provide the - // uncompressed SHA256; the cache will take care of - // uncompressing before checksumming. - baseURL += "?sha256=" + meta.Images.OpenStack.UncompressedSHA256 - - // Check that we have generated a valid URL - _, err = url.ParseRequestURI(baseURL) - if err != nil { - return "", err - } - - return baseURL, nil -} - // GenerateOpenStackImageName returns Glance image name for instances. func GenerateOpenStackImageName(rhcosImage, infraID string) (imageName string, isURL bool) { // Here we check whether rhcosImage is a URL or not. If this is the first case, it means that Glance image diff --git a/pkg/rhcos/qemu.go b/pkg/rhcos/qemu.go deleted file mode 100644 index f0c0026cc80..00000000000 --- a/pkg/rhcos/qemu.go +++ /dev/null @@ -1,43 +0,0 @@ -package rhcos - -import ( - "context" - "net/url" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" -) - -// QEMU fetches the URL of the Red Hat Enterprise Linux CoreOS release. -func QEMU(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - base, err := url.Parse(meta.BaseURI) - if err != nil { - return "", err - } - - relQEMU, err := url.Parse(meta.Images.QEMU.Path) - if err != nil { - return "", err - } - - baseURL := base.ResolveReference(relQEMU).String() - - // Attach sha256 checksum to the URL. Always provide the - // uncompressed SHA256; the cache will take care of - // uncompressing before checksumming. - baseURL += "?sha256=" + meta.Images.QEMU.UncompressedSHA256 - - // Check that we have generated a valid URL - _, err = url.ParseRequestURI(baseURL) - if err != nil { - return "", err - } - - return baseURL, nil -} diff --git a/pkg/rhcos/vmware.go b/pkg/rhcos/vmware.go deleted file mode 100644 index d3b1a368f3f..00000000000 --- a/pkg/rhcos/vmware.go +++ /dev/null @@ -1,43 +0,0 @@ -package rhcos - -import ( - "context" - "net/url" - - "github.com/pkg/errors" - - "github.com/openshift/installer/pkg/types" -) - -// VMware fetches the URL of the Red Hat Enterprise Linux CoreOS release. -func VMware(ctx context.Context, arch types.Architecture) (string, error) { - meta, err := fetchRHCOSBuild(ctx, arch) - if err != nil { - return "", errors.Wrap(err, "failed to fetch RHCOS metadata") - } - - base, err := url.Parse(meta.BaseURI) - if err != nil { - return "", err - } - - image, err := url.Parse(meta.Images.VMware.Path) - if err != nil { - return "", err - } - - baseURL := base.ResolveReference(image).String() - - // Attach sha256 checksum to the URL. Always provide the - // uncompressed SHA256; the cache will take care of - // uncompressing before checksumming. - baseURL += "?sha256=" + meta.Images.VMware.SHA256 - - // Check that we have generated a valid URL - _, err = url.ParseRequestURI(baseURL) - if err != nil { - return "", err - } - - return baseURL, nil -}