Merge pull request #588 from amazonlinux/fancy-models

Allow for a separate API model per variant

Dockerfile

@@ -16,10 +16,33 @@ COPY --chown=1000:1000 --from=sdk /tmp /cache
 # Ensure the ARG variables are used in the layer to prevent reuse by other builds.
 COPY --chown=1000:1000 .dockerignore /cache/.${PACKAGE}.${ARCH}
 
+# Some builds need to modify files in the source directory, for example Rust
+# software using build.rs to generate code.  The source directory is mounted in
+# using "--mount=source" which is owned by root, and we need to modify it as
+# the builder user.  To get around this, we can use a "cache" mount, which we
+# just won't share or reuse.  We mount a cache into the location we need to
+# change, and in some cases, set up symlinks so that it looks like a normal
+# part of the source tree.  (This is like a tmpfs mount, but cache mounts have
+# more flexibility - you can specify a source to set them up beforehand,
+# specify uid/gid, etc.)  This cache is also variant-specific (in addition to
+# package and arch, like the one above) for cases where we need to build
+# differently per variant; the cache will be empty if you change
+# BUILDSYS_VARIANT.
+FROM scratch AS variantcache
+ARG PACKAGE
+ARG ARCH
+ARG VARIANT
+# We can't create directories via RUN in a scratch container, so take an existing one.
+COPY --chown=1000:1000 --from=sdk /tmp /variantcache
+# Ensure the ARG variables are used in the layer to prevent reuse by other builds.
+COPY --chown=1000:1000 .dockerignore /variantcache/.${PACKAGE}.${ARCH}.${VARIANT}
+
 FROM sdk AS rpmbuild
 ARG PACKAGE
 ARG ARCH
 ARG NOCACHE
+ARG VARIANT
+ENV VARIANT=${VARIANT}
 WORKDIR /home/builder
 
 USER builder
@@ -49,9 +72,12 @@ RUN --mount=target=/host \
         --nogpgcheck \
         builddep rpmbuild/SPECS/${PACKAGE}.spec
 
+# We use the "nocache" writable space to generate code where necessary, like
+# the variant-specific models.
 USER builder
 RUN --mount=source=.cargo,target=/home/builder/.cargo \
     --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \
+    --mount=type=cache,target=/home/builder/rpmbuild/BUILD/workspaces/models/current,from=variantcache,source=/variantcache \
     --mount=source=workspaces,target=/home/builder/rpmbuild/BUILD/workspaces \
     rpmbuild -ba --clean rpmbuild/SPECS/${PACKAGE}.spec
 

README.md

@@ -216,7 +216,7 @@ timezone = "America/Thunder_Bay"
 
 Here we'll describe each setting you can change.
 
-**Note:** You can see the [default values](workspaces/api/storewolf/defaults.toml) for any settings that have defaults.
+**Note:** You can see the default values (for any settings that have defaults) by looking at the `defaults.toml` for your Thar variant under [models](workspaces/models/).
 
 When you're sending settings to the API, or receiving settings from the API, they're in a structured JSON format.
 This allows allow modification of any number of keys at once.

packages/workspaces/Cargo.toml

@@ -6,6 +6,7 @@ publish = false
 build = "build.rs"
 
 [package.metadata.build-package]
+variant-sensitive = true
 source-groups = [
     "api",
     "growpart",

tools/buildsys/src/builder.rs

@@ -21,17 +21,25 @@ impl PackageBuilder {
     pub(crate) fn build(package: &str) -> Result<(Self)> {
         let arch = getenv("BUILDSYS_ARCH")?;
 
+        // We do *not* want to rebuild most packages when the variant changes, becauses most aren't
+        // affected; packages that care about variant should "echo cargo:rerun-if-env-changed=VAR"
+        // themselves in the package's spec file.
+        let var = "BUILDSYS_VARIANT";
+        let variant = env::var(var).context(error::Environment { var })?;
+
         let target = "package";
         let build_args = format!(
             "--build-arg PACKAGE={package} \
-             --build-arg ARCH={arch}",
+             --build-arg ARCH={arch} \
+             --build-arg VARIANT={variant}",
             package = package,
             arch = arch,
+            variant = variant,
         );
         let tag = format!(
             "buildsys-pkg-{package}-{arch}",
             package = package,
-            arch = arch
+            arch = arch,
         );
 
         build(&target, &build_args, &tag)?;

tools/buildsys/src/main.rs

@@ -98,6 +98,13 @@ fn build_package() -> Result<()> {
     let manifest =
         ManifestInfo::new(manifest_dir.join(manifest_file)).context(error::ManifestParse)?;
 
+    // if manifest has package.metadata.build-package.variant-specific = true, then println rerun-if-env-changed
+    if let Some(sensitive) = manifest.variant_sensitive() {
+        if sensitive {
+            println!("cargo:rerun-if-env-changed=BUILDSYS_VARIANT");
+        }
+    }
+
     if let Some(files) = manifest.external_files() {
         LookasideCache::fetch(&files).context(error::ExternalFileFetch)?;
     }

tools/buildsys/src/manifest.rs

@@ -70,6 +70,11 @@ impl ManifestInfo {
         self.build_package().and_then(|b| b.external_files.as_ref())
     }
 
+    /// Convenience method to find whether the package is sensitive to variant changes.
+    pub(crate) fn variant_sensitive(&self) -> Option<bool> {
+        self.build_package().and_then(|b| b.variant_sensitive)
+    }
+
     /// Convenience method to return the list of included packages.
     pub(crate) fn included_packages(&self) -> Option<&Vec<String>> {
         self.build_variant()
@@ -110,6 +115,7 @@ struct Metadata {
 pub(crate) struct BuildPackage {
     pub(crate) source_groups: Option<Vec<PathBuf>>,
     pub(crate) external_files: Option<Vec<ExternalFile>>,
+    pub(crate) variant_sensitive: Option<bool>,
 }
 
 #[derive(Deserialize, Debug)]

workspaces/Cargo.toml

@@ -21,6 +21,8 @@ members = [
     "api/migration/migrations/v0.1/borkseed",
     "api/migration/migrations/v0.1/host-containers-version",
 
+    "models",
+
     "preinit/laika",
 
     "updater/block-party",

workspaces/api/README.md

@@ -49,7 +49,7 @@ On first boot, [storewolf](#storewolf) hasn’t run yet, so there’s no data st
 
 storewolf owns the creation and initial population of the data store.
 
-storewolf ensures the defined [default values](storewolf/defaults.toml) are populated in the data store.
+storewolf ensures the default values (defined in the `defaults.toml` for [your variant](../models)) are populated in the data store.
 First, it has to create the data store directories and symlinks if they don’t exist.
 Then, it goes key-by-key through the defaults, and if a key isn’t already set, sets it with the default value.
 
@@ -107,7 +107,7 @@ This service sends a commit request to the API, which moves all the pending sett
 
 Further docs:
 * [thar-be-settings](thar-be-settings/), the tool settings-applier uses
-* [defaults.toml](storewolf/defaults.toml), which defines our configuration files and services
+* The `defaults.toml` files for each [variant](../models), which define our configuration files and services
 
 This is a simple startup service that runs `thar-be-settings --all` to write out all of the configuration files that are based on our settings.
 

workspaces/api/apiserver/Cargo.toml

@@ -8,20 +8,16 @@ build = "build.rs"
 
 [dependencies]
 actix-web = { version = "1.0.5", default-features = false, features = ["uds"] }
-base64 = "0.10"
-lazy_static = "1.2"
 libc = "0.2"
 log = "0.4"
+models = { path = "../../models" }
 nix = "0.15.0"
 percent-encoding = "2.1"
-regex = "1.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 simplelog = "0.7"
 snafu = "0.5"
 systemd = { version = "0.4.0", default-features = false, features = [], optional = true }
-toml = "0.5"
-url = "2.1"
 walkdir = "2.2"
 
 [features]
@@ -34,3 +30,4 @@ cargo-readme = "3.1"
 
 [dev-dependencies]
 maplit = "1.0"
+toml = "0.5"

workspaces/api/apiserver/README.md

@@ -34,7 +34,7 @@ Requests are directed by `server::router`.
 ### Model
 
 The API is driven by a data model (similar to a schema) defined in Rust.
-See the 'model' module.
+See the 'models' workspace.
 All input is deserialized into model types, and all output is serialized from model types, so we can be more confident that data is in the format we expect.
 
 The data model describes system settings, services using those settings, and configuration files used by those services.

workspaces/api/apiserver/src/lib.rs

@@ -31,7 +31,7 @@ Requests are directed by `server::router`.
 ## Model
 
 The API is driven by a data model (similar to a schema) defined in Rust.
-See the 'model' module.
+See the 'models' workspace.
 All input is deserialized into model types, and all output is serialized from model types, so we can be more confident that data is in the format we expect.
 
 The data model describes system settings, services using those settings, and configuration files used by those services.
@@ -78,8 +78,6 @@ See `../../apiclient/README.md` for client examples.
 extern crate log;
 
 pub mod datastore;
-pub mod model;
-pub mod modeled_types;
 pub mod server;
 
 pub use server::serve;

workspaces/api/apiserver/src/server/controller.rs

@@ -12,8 +12,8 @@ use crate::datastore::serialization::to_pairs;
 use crate::datastore::{
     deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value,
 };
-use crate::model::{ConfigurationFiles, Services, Settings};
 use crate::server::error::{self, Result};
+use model::{ConfigurationFiles, Services, Settings};
 
 /// Build a Settings based on pending data in the datastore; the Settings will be empty if there
 /// are no pending settings.
@@ -339,8 +339,8 @@ mod test {
     use super::*;
     use crate::datastore::memory::MemoryDataStore;
     use crate::datastore::{Committed, DataStore, Key, KeyType};
-    use crate::model::Service;
     use maplit::{hashmap, hashset};
+    use model::Service;
     use std::convert::TryInto;
 
     #[test]

workspaces/api/apiserver/src/server/mod.rs

@@ -12,8 +12,8 @@ use std::path::Path;
 use std::sync;
 
 use crate::datastore::{Committed, FilesystemDataStore, Key, Value};
-use crate::model::{ConfigurationFiles, Services, Settings};
 use error::Result;
+use model::{ConfigurationFiles, Services, Settings};
 
 use nix::unistd::{chown, Gid};
 use std::fs::set_permissions;
@@ -122,10 +122,10 @@ where
 fn get_settings(
     query: web::Query<HashMap<String, String>>,
     data: web::Data<SharedDataStore>,
-) -> Result<Settings> {
+) -> Result<SettingsResponse> {
     let datastore = data.ds.read().ok().context(error::DataStoreLock)?;
 
-    if let Some(keys_str) = query.get("keys") {
+    let settings = if let Some(keys_str) = query.get("keys") {
         let keys = comma_separated("keys", keys_str)?;
         controller::get_settings_keys(&*datastore, &keys, Committed::Live)
     } else if let Some(prefix_str) = query.get("prefix") {
@@ -136,7 +136,9 @@ fn get_settings(
         controller::get_settings_prefix(&*datastore, prefix_str, Committed::Live)
     } else {
         controller::get_settings(&*datastore, Committed::Live)
-    }
+    }?;
+
+    Ok(SettingsResponse(settings))
 }
 
 /// Apply the requested settings to the pending data store
@@ -150,9 +152,10 @@ fn patch_settings(
 }
 
 /// Return any settings that have been received but not committed
-fn get_pending_settings(data: web::Data<SharedDataStore>) -> Result<Settings> {
+fn get_pending_settings(data: web::Data<SharedDataStore>) -> Result<SettingsResponse> {
     let datastore = data.ds.read().ok().context(error::DataStoreLock)?;
-    controller::get_pending_settings(&*datastore)
+    let settings = controller::get_pending_settings(&*datastore)?;
+    Ok(SettingsResponse(settings))
 }
 
 /// Delete any settings that have been received but not committed
@@ -338,8 +341,9 @@ macro_rules! impl_responder_for {
     )
 }
 
-// This lets us respond from our handler methods with a Settings (or Result<Settings>)
-impl_responder_for!(Settings, self, self);
+/// This lets us respond from our handler methods with a Settings (or Result<Settings>)
+struct SettingsResponse(Settings);
+impl_responder_for!(SettingsResponse, self, self.0);
 
 /// This lets us respond from our handler methods with a HashMap (or Result<HashMap>) for metadata
 struct MetadataResponse(HashMap<String, Value>);

workspaces/api/host-containers/Cargo.toml

@@ -11,6 +11,7 @@ apiclient = { path = "../apiclient" }
 apiserver = { path = "../apiserver" }
 http = "0.1"
 log = "0.4"
+models = { path = "../../models" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 simplelog = "0.7"

workspaces/api/host-containers/src/main.rs

@@ -22,8 +22,7 @@ use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 use std::str::FromStr;
 
-use apiserver::model;
-use apiserver::modeled_types::Identifier;
+use model::modeled_types::Identifier;
 
 // FIXME Get from configuration in the future
 const DEFAULT_API_SOCKET: &str = "/run/api.sock";

workspaces/api/servicedog/Cargo.toml

@@ -11,6 +11,7 @@ apiclient = { path = "../apiclient" }
 apiserver = { path = "../apiserver" }
 http = "0.1"
 log = "0.4"
+models = { path = "../../models" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 simplelog = "0.7"