stages/ostree.deploy.container: add aleph file

osbuild.spec

@@ -1,7 +1,7 @@
 %global         forgeurl https://github.com/osbuild/osbuild
 %global         selinuxtype targeted
 
-Version:        98
+Version:        99
 
 %forgemeta
 

osbuild/__init__.py

@@ -10,7 +10,7 @@
 
 from .pipeline import Manifest, Pipeline, Stage
 
-__version__ = "98"
+__version__ = "99"
 
 __all__ = [
     "Manifest",

osbuild/buildroot.py

@@ -196,7 +196,7 @@ def run(self, argv, monitor, timeout=None, binds=None, readonly_binds=None, extr
 
         # Import directories from the caller-provided root.
         imports = ["usr"]
-        if self.mount_boot:
+        if True:
             imports.insert(0, "boot")
 
         for p in imports:

osbuild/objectstore.py

@@ -287,10 +287,18 @@ def init(self):
         # with just /usr mounted from the host
         usr = os.path.join(root, "usr")
         os.makedirs(usr)
+        boot = os.path.join(root, "boot")
+        os.makedirs(boot)
+        etc = os.path.join(root, "etc")
+        os.makedirs(etc)
 
         # ensure / is read-only
         mount(root, root)
+        mount("/etc", etc)
         mount("/usr", usr)
+        mount("/boot", boot)
+
+
 
     @property
     def tree(self) -> os.PathLike:

osbuild/util/ostree.py

@@ -8,6 +8,7 @@
 import typing
 # pylint doesn't understand the string-annotation below
 from typing import Any, List  # pylint: disable=unused-import
+import glob
 
 from osbuild.util.rhsm import Subscriptions
 
@@ -192,18 +193,16 @@ def pull_local(source_repo: PathLike, target_repo: PathLike, remote: str, ref: s
         *extra_args,
         repo=target_repo)
 
-
 def cli(*args, _input=None, **kwargs):
     """Thin wrapper for running the ostree CLI"""
     args = list(args) + [f'--{k}={v}' for k, v in kwargs.items()]
     print("ostree " + " ".join(args), file=sys.stderr)
-    subprocess.run(["ostree"] + args,
+    return subprocess.run(["ostree"] + args,
                    encoding="utf8",
-                   stdout=sys.stderr,
+                   stdout=subprocess.PIPE,
                    input=_input,
                    check=True)
 
-
 def parse_input_commits(commits):
     """Parse ostree input commits and return the repo path and refs specified"""
     data = commits["data"]
@@ -215,15 +214,24 @@ def parse_input_commits(commits):
 def deployment_path(root: PathLike, osname: str, ref: str, serial: int):
     """Return the path to a deployment given the parameters"""
 
-    base = os.path.join(root, "ostree")
+    if osname == "" and ref == "" and serial == None:
+        filenames =  glob.glob(root + '/ostree/deploy/*/deploy/*.0', recursive=True)
+        if len(filenames) < 1:
+            raise ValueError("Cound not find deployment")
+        elif len(filenames) > 1:
+            raise ValueError("More than one deployment found")
+        return filenames[0]
+
+    else:
+        base = os.path.join(root, "ostree")
 
-    repo = os.path.join(base, "repo")
-    stateroot = os.path.join(base, "deploy", osname)
+        repo = os.path.join(base, "repo")
+        stateroot = os.path.join(base, "deploy", osname)
 
-    commit = rev_parse(repo, ref)
-    sysroot = f"{stateroot}/deploy/{commit}.{serial}"
+        commit = rev_parse(repo, ref)
+        sysroot = f"{stateroot}/deploy/{commit}.{serial}"
 
-    return sysroot
+        return sysroot
 
 
 class PasswdLike:

setup.py

@@ -2,7 +2,7 @@
 
 setuptools.setup(
     name="osbuild",
-    version="98",
+    version="99",
     description="A build system for OS images",
     packages=["osbuild", "osbuild.formats", "osbuild.util"],
     license='Apache-2.0',

stages/org.osbuild.ostree.coreos.aleph

@@ -0,0 +1,172 @@
+#!/usr/bin/python3
+"""
+Create aleph version file for the deployment.
+"""
+
+
+import os
+import sys
+
+import osbuild.api
+from osbuild.util import ostree
+
+import json
+
+CAPABILITIES = ["CAP_MAC_ADMIN"]
+
+
+SCHEMA_2 = """
+"options": {
+  "additionalProperties": false,
+  "properties": {
+    "coreos_compat": {
+      "description": "boolean to allow for CoreOS aleph version backwards compatibility",
+      "type": "boolean"
+    },
+    "deployment": {
+      "additionalProperties": false,
+      "required": ["osname", "ref"],
+      "properties": {
+        "osname": {
+          "description": "Name of the stateroot to be used in the deployment",
+          "type": "string"
+        },
+        "ref": {
+          "description": "OStree ref to create and use for deployment",
+          "type": "string"
+        },
+        "serial": {
+          "description": "The deployment serial (usually '0')",
+          "type": "number",
+          "default": 0
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def aleph_commit(tree, imgref):
+    extra_args = []
+    extra_args.append("--print-metadata-key=version")
+
+    aleph_version = ostree.cli("show", f"--repo={tree}/ostree/repo", "ostree/1/1/0", *extra_args).stdout.rstrip().strip('\'')
+    aleph_ref = imgref
+    # get the commit by parsing the revision of the deployment
+    aleph_ostree_commit = ostree.rev_parse(tree + "/ostree/repo", "ostree/1/1/0")
+
+    aleph_version_data = {
+        "osbuild-version": osbuild.__version__,
+        "version": aleph_version,
+        "ref": aleph_ref,
+        "ostree-commit": aleph_ostree_commit
+    }
+
+    return aleph_version_data
+
+
+def aleph_container(tree, imgref):
+    # extract the image name from the imgref
+    imgref_list = imgref.split(':')
+    if imgref_list[0] in ["ostree-remote-registry", "ostree-remote-image"]:
+        img_name = ':'.join(imgref_list[2:])
+    elif imgref_list[0] in ["ostree-image-signed", "ostree-unverified-registry"]:
+        img_name = ':'.join(imgref_list[1:])
+
+    img_name = img_name.lstrip('docker://')
+
+    extra_args = []
+    extra_args.append(f"--repo={tree}/ostree/repo")
+    extra_args.append(f"registry:{img_name}")
+
+    container_data_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip()
+    container_data = json.loads(container_data_json)
+
+    extra_args.append("--config")
+    container_data_config_json = ostree.cli("container", "image", "metadata", *extra_args).stdout.rstrip()
+    container_data_config = json.loads(container_data_config_json)
+
+    aleph_digest = container_data['config']['digest']
+    aleph_ref = f"docker://{imgref}"
+    aleph_version = container_data_config['config']['Labels']['org.opencontainers.image.version']
+    aleph_container_image = container_data_config['config']['Labels']
+
+    aleph_version_data = {
+        "osbuild-version": osbuild.__version__,
+        "ref": aleph_ref,
+        "build": aleph_version,
+        "version": aleph_version,
+        "container-image": {
+            "image-name": img_name,
+            "image-digest": aleph_digest,
+            "image-labels": aleph_container_image
+        }
+    }
+
+    # the 'ostree.commit' label will be optional in the future so
+    # prevent hard failing if key is not found
+    aleph_ostree_commit = container_data_config['config']['Labels'].get('ostree.commit')
+    if aleph_ostree_commit is not None:
+        aleph_version_data["ostree-commit"] = aleph_ostree_commit
+
+    return aleph_version_data
+
+
+def main(tree, options):
+    coreos_compat = options.get("coreos_compat", False)
+    dep = options.get("deployment", None)
+    osname = ""
+    ref = ""
+    serial = None
+    origin = ""
+
+    # if deployment is specified, use it to find the origin file.
+    # otherwise, autodetect the only deployment found in the tree.
+    if dep is not None:
+        osname = dep.get("osname", "")
+        ref = dep.get("ref", "")
+        serial = dep.get("serial", 0)
+
+    origin = ostree.deployment_path(tree, osname, ref, serial) + ".origin"
+    # use the origin file to find the target_imgref and dictate whether
+    # the system was deployed from an container image or ostree commit.
+    # example container case: container-image-reference=ostree-remote-image:fedora:docker://quay.io/fedora/fedora-coreos:stable
+    # example ostree commit case: refspec=fedora:fedora/x86_64/coreos/stable
+    deploy_type = ""
+    imgref = ""
+    data = {}
+    with open(origin) as f:
+        for line in f:
+            separated_line = line.split("=")
+            if separated_line[0] == "container-image-reference":
+                deploy_type = "container"
+                imgref = separated_line[1].rstrip()
+                break
+            elif separated_line[0] == "refspec":
+                deploy_type = "ostree_commit"
+                imgref = separated_line[1].rstrip()
+                break
+    if deploy_type == "container":
+        data = aleph_container(tree, imgref)
+    elif deploy_type == "ostree_commit":
+        data = aleph_commit(tree, imgref)
+    else:
+        raise ValueError("Could not find 'container-image-reference' or 'refspec' in origin file")
+
+    # write the data out to the file
+    filename = ".aleph-version.json"
+    with open(os.path.join(tree, filename), "w") as f:
+        json.dump(data, f, indent=4, sort_keys=True)
+        f.write("\n")
+
+    # create a symlink for backwards compatibility with CoreOS
+    if coreos_compat:
+      os.symlink(filename, os.path.join(tree, ".coreos-aleph-version.json"))
+
+
+if __name__ == '__main__':
+    stage_args = osbuild.api.arguments()
+    r = main(stage_args["tree"],
+             stage_args["options"])
+    sys.exit(r)

test/data/manifests/fedora-coreos-container.mpp.yaml

@@ -25,16 +25,22 @@ mpp-define-image:
         size: 4194304
         type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4
         uuid: CA7D7CCB-63ED-4C53-861C-1742536059CC
+# sources:
+#   org.osbuild.ostree:
+#     items:
+#       2c39e9b6a6b26226b626f4daf815d5307a96b2cd57463e94f7c618e312a61980:
+#         remote:
+#           url: https://kojipkgs.fedoraproject.org/ostree/repo/
 pipelines:
   - mpp-import-pipelines:
       path: fedora-vars.ipp.yaml
-  - mpp-import-pipeline:
-      path: fedora-build-v2.ipp.yaml
-      id: build
+  # - mpp-import-pipeline:
+  #     path: fedora-build-v2.ipp.yaml
+  #     id: build
     runner:
       mpp-format-string: org.osbuild.fedora{release}
   - name: image-tree
-    build: name:build
+    # build: name:build
     source-epoch: 1659397331
     stages:
       - type: org.osbuild.ostree.init-fs
@@ -75,6 +81,32 @@ pipelines:
               images:
                 - source: registry.gitlab.com/redhat/services/products/image-builder/ci/images/fedora-coreos
                   tag: stable
+      - type: org.osbuild.ostree.coreos.aleph
+        options:
+          coreos_compat: true
+      # - type: org.osbuild.ostree.deploy
+      #   options:
+      #     osname: fedora-coreos
+      #     remote: fedora
+      #     mounts:
+      #       - /boot
+      #       - /boot/efi
+      #     kernel_opts:
+      #       - rw
+      #       - console=tty0
+      #       - console=ttyS0
+      #       - ignition.platform.id=qemu
+      #       - '$ignition_firstboot'
+      #   inputs:
+      #     commits:
+      #       type: org.osbuild.ostree
+      #       origin: org.osbuild.source
+      #       references:
+      #         "2c39e9b6a6b26226b626f4daf815d5307a96b2cd57463e94f7c618e312a61980":
+      #           ref: fedora/x86_64/coreos/stable
+      # - type: org.osbuild.ostree.coreos.aleph
+      #   options:
+      #     coreos_compat: true
       - type: org.osbuild.ostree.selinux
         options:
           deployment:
@@ -94,7 +126,7 @@ pipelines:
           greenboot: false
           ignition: true
   - name: image
-    build: name:build
+    # build: name:build
     stages:
       - type: org.osbuild.truncate
         options:
@@ -218,7 +250,7 @@ pipelines:
               mpp-format-int: '{image.layout[''boot''].index}'
             path: /grub2
   - name: qcow2
-    build: name:build
+    # build: name:build
     stages:
       - type: org.osbuild.qemu
         inputs: