From 93b87ca16aa2af4815413d6f444a31816294d034 Mon Sep 17 00:00:00 2001 From: Tom Kirchner Date: Wed, 11 Dec 2019 15:36:24 -0800 Subject: [PATCH] Allow for a separate API model per variant Pull #578 added support for build variants to support different features and use cases, starting with aws-k8s and aws-dev. These variants use different software, and therefore need different configuration. This change allows for having a different API model (and default values) per variant. The model was moved to workspaces/models. This crate actually contains the model for each variant in separate subdirectories. Cargo's `build.rs` is used to symlink in the proper source code before build. The "proper" source code is chosen using the VARIANT environment variable. The build system passes this through using the value of BUILDSYS_VARIANT, which is set in the top-level Makefile.toml and can be overridden on the command-line. For local cargo builds, the developer must set the VARIANT environment variable; we don't currently want to choose a default model. To start out, the aws-k8s and aws-dev models (and related default values) are copies, with aws-dev having the Kubernetes-related values removed. The next step is to figure out an appropriate method for sharing structure; until then, we'll need to duplicate most changes. --- Dockerfile | 26 ++ README.md | 2 +- packages/workspaces/Cargo.toml | 1 + tools/buildsys/src/builder.rs | 12 +- tools/buildsys/src/main.rs | 7 + tools/buildsys/src/manifest.rs | 6 + workspaces/Cargo.lock | 23 +- workspaces/Cargo.toml | 2 + workspaces/api/README.md | 4 +- workspaces/api/apiserver/Cargo.toml | 7 +- workspaces/api/apiserver/README.md | 2 +- workspaces/api/apiserver/src/lib.rs | 4 +- .../api/apiserver/src/server/controller.rs | 4 +- workspaces/api/apiserver/src/server/mod.rs | 20 +- workspaces/api/host-containers/Cargo.toml | 1 + workspaces/api/host-containers/src/main.rs | 3 +- workspaces/api/servicedog/Cargo.toml | 1 + workspaces/api/servicedog/src/main.rs | 1 - workspaces/api/storewolf/Cargo.toml | 1 + workspaces/api/storewolf/src/main.rs | 7 +- workspaces/api/sundog/Cargo.toml | 1 + workspaces/api/sundog/src/main.rs | 1 - workspaces/api/thar-be-settings/Cargo.toml | 1 + workspaces/api/thar-be-settings/src/config.rs | 2 - .../api/thar-be-settings/src/service.rs | 1 - .../api/thar-be-settings/src/settings.rs | 2 - .../api/thar-be-settings/src/template.rs | 2 - workspaces/deny.toml | 1 + workspaces/models/.gitignore | 1 + workspaces/models/Cargo.toml | 18 + workspaces/models/README.md | 31 ++ workspaces/models/aws-dev/defaults.toml | 70 ++++ workspaces/models/aws-dev/model.rs | 110 ++++++ workspaces/models/aws-dev/modeled_types.rs | 321 ++++++++++++++++++ .../aws-k8s}/defaults.toml | 0 .../apiserver/src => models/aws-k8s}/model.rs | 2 + .../src => models/aws-k8s}/modeled_types.rs | 0 workspaces/models/build.rs | 57 ++++ workspaces/models/current/.keep | 3 + 39 files changed, 715 insertions(+), 43 deletions(-) create mode 100644 workspaces/models/.gitignore create mode 100644 workspaces/models/Cargo.toml create mode 100644 workspaces/models/README.md create mode 100644 workspaces/models/aws-dev/defaults.toml create mode 100644 workspaces/models/aws-dev/model.rs create mode 100644 workspaces/models/aws-dev/modeled_types.rs rename workspaces/{api/storewolf => models/aws-k8s}/defaults.toml (100%) rename workspaces/{api/apiserver/src => models/aws-k8s}/model.rs (99%) rename workspaces/{api/apiserver/src => models/aws-k8s}/modeled_types.rs (100%) create mode 100644 workspaces/models/build.rs create mode 100644 workspaces/models/current/.keep diff --git a/Dockerfile b/Dockerfile index 899654e6b81..4fa40bff0d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,10 +16,33 @@ COPY --chown=1000:1000 --from=sdk /tmp /cache # Ensure the ARG variables are used in the layer to prevent reuse by other builds. COPY --chown=1000:1000 .dockerignore /cache/.${PACKAGE}.${ARCH} +# Some builds need to modify files in the source directory, for example Rust +# software using build.rs to generate code. The source directory is mounted in +# using "--mount=source" which is owned by root, and we need to modify it as +# the builder user. To get around this, we can use a "cache" mount, which we +# just won't share or reuse. We mount a cache into the location we need to +# change, and in some cases, set up symlinks so that it looks like a normal +# part of the source tree. (This is like a tmpfs mount, but cache mounts have +# more flexibility - you can specify a source to set them up beforehand, +# specify uid/gid, etc.) This cache is also variant-specific (in addition to +# package and arch, like the one above) for cases where we need to build +# differently per variant; the cache will be empty if you change +# BUILDSYS_VARIANT. +FROM scratch AS variantcache +ARG PACKAGE +ARG ARCH +ARG VARIANT +# We can't create directories via RUN in a scratch container, so take an existing one. +COPY --chown=1000:1000 --from=sdk /tmp /variantcache +# Ensure the ARG variables are used in the layer to prevent reuse by other builds. +COPY --chown=1000:1000 .dockerignore /variantcache/.${PACKAGE}.${ARCH}.${VARIANT} + FROM sdk AS rpmbuild ARG PACKAGE ARG ARCH ARG NOCACHE +ARG VARIANT +ENV VARIANT=${VARIANT} WORKDIR /home/builder USER builder @@ -49,9 +72,12 @@ RUN --mount=target=/host \ --nogpgcheck \ builddep rpmbuild/SPECS/${PACKAGE}.spec +# We use the "nocache" writable space to generate code where necessary, like +# the variant-specific models. USER builder RUN --mount=source=.cargo,target=/home/builder/.cargo \ --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \ + --mount=type=cache,target=/home/builder/rpmbuild/BUILD/workspaces/models/current,from=variantcache,source=/variantcache \ --mount=source=workspaces,target=/home/builder/rpmbuild/BUILD/workspaces \ rpmbuild -ba --clean rpmbuild/SPECS/${PACKAGE}.spec diff --git a/README.md b/README.md index fb6fec42a6f..adf7f12f0fd 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ timezone = "America/Thunder_Bay" Here we'll describe each setting you can change. -**Note:** You can see the [default values](workspaces/api/storewolf/defaults.toml) for any settings that have defaults. +**Note:** You can see the default values (for any settings that have defaults) by looking at the `defaults.toml` for your Thar variant under [models](workspaces/models/). When you're sending settings to the API, or receiving settings from the API, they're in a structured JSON format. This allows allow modification of any number of keys at once. diff --git a/packages/workspaces/Cargo.toml b/packages/workspaces/Cargo.toml index 64896bcba07..f0429ddb402 100644 --- a/packages/workspaces/Cargo.toml +++ b/packages/workspaces/Cargo.toml @@ -6,6 +6,7 @@ publish = false build = "build.rs" [package.metadata.build-package] +variant-sensitive = true source-groups = [ "api", "growpart", diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index f4de8e243f9..4903ff401a7 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -21,17 +21,25 @@ impl PackageBuilder { pub(crate) fn build(package: &str) -> Result<(Self)> { let arch = getenv("BUILDSYS_ARCH")?; + // We do *not* want to rebuild most packages when the variant changes, becauses most aren't + // affected; packages that care about variant should "echo cargo:rerun-if-env-changed=VAR" + // themselves in the package's spec file. + let var = "BUILDSYS_VARIANT"; + let variant = env::var(var).context(error::Environment { var })?; + let target = "package"; let build_args = format!( "--build-arg PACKAGE={package} \ - --build-arg ARCH={arch}", + --build-arg ARCH={arch} \ + --build-arg VARIANT={variant}", package = package, arch = arch, + variant = variant, ); let tag = format!( "buildsys-pkg-{package}-{arch}", package = package, - arch = arch + arch = arch, ); build(&target, &build_args, &tag)?; diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 8aacfc956a5..77ff8759922 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -98,6 +98,13 @@ fn build_package() -> Result<()> { let manifest = ManifestInfo::new(manifest_dir.join(manifest_file)).context(error::ManifestParse)?; + // if manifest has package.metadata.build-package.variant-specific = true, then println rerun-if-env-changed + if let Some(sensitive) = manifest.variant_sensitive() { + if sensitive { + println!("cargo:rerun-if-env-changed=BUILDSYS_VARIANT"); + } + } + if let Some(files) = manifest.external_files() { LookasideCache::fetch(&files).context(error::ExternalFileFetch)?; } diff --git a/tools/buildsys/src/manifest.rs b/tools/buildsys/src/manifest.rs index deebbf5441a..55d8f34b400 100644 --- a/tools/buildsys/src/manifest.rs +++ b/tools/buildsys/src/manifest.rs @@ -70,6 +70,11 @@ impl ManifestInfo { self.build_package().and_then(|b| b.external_files.as_ref()) } + /// Convenience method to find whether the package is sensitive to variant changes. + pub(crate) fn variant_sensitive(&self) -> Option { + self.build_package().and_then(|b| b.variant_sensitive) + } + /// Convenience method to return the list of included packages. pub(crate) fn included_packages(&self) -> Option<&Vec> { self.build_variant() @@ -110,6 +115,7 @@ struct Metadata { pub(crate) struct BuildPackage { pub(crate) source_groups: Option>, pub(crate) external_files: Option>, + pub(crate) variant_sensitive: Option, } #[derive(Deserialize, Debug)] diff --git a/workspaces/Cargo.lock b/workspaces/Cargo.lock index dca4a2e4528..75ac3523314 100644 --- a/workspaces/Cargo.lock +++ b/workspaces/Cargo.lock @@ -271,22 +271,19 @@ name = "apiserver" version = "0.1.0" dependencies = [ "actix-web 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "nix 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "snafu 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "systemd 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1017,6 +1014,7 @@ dependencies = [ "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1414,6 +1412,19 @@ dependencies = [ "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "models" +version = "0.1.0" +dependencies = [ + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", + "snafu 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "moondog" version = "0.1.0" @@ -2110,6 +2121,7 @@ dependencies = [ "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2272,6 +2284,7 @@ dependencies = [ "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "data_store_version 0.1.0", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "snafu 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2321,6 +2334,7 @@ dependencies = [ "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2432,6 +2446,7 @@ dependencies = [ "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "models 0.1.0", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/workspaces/Cargo.toml b/workspaces/Cargo.toml index 444c7c53b92..09695f7307d 100644 --- a/workspaces/Cargo.toml +++ b/workspaces/Cargo.toml @@ -21,6 +21,8 @@ members = [ "api/migration/migrations/v0.1/borkseed", "api/migration/migrations/v0.1/host-containers-version", + "models", + "preinit/laika", "updater/block-party", diff --git a/workspaces/api/README.md b/workspaces/api/README.md index 00dadcd7941..fa77c27f55f 100644 --- a/workspaces/api/README.md +++ b/workspaces/api/README.md @@ -49,7 +49,7 @@ On first boot, [storewolf](#storewolf) hasn’t run yet, so there’s no data st storewolf owns the creation and initial population of the data store. -storewolf ensures the defined [default values](storewolf/defaults.toml) are populated in the data store. +storewolf ensures the default values (defined in the `defaults.toml` for [your variant](../models)) are populated in the data store. First, it has to create the data store directories and symlinks if they don’t exist. Then, it goes key-by-key through the defaults, and if a key isn’t already set, sets it with the default value. @@ -107,7 +107,7 @@ This service sends a commit request to the API, which moves all the pending sett Further docs: * [thar-be-settings](thar-be-settings/), the tool settings-applier uses -* [defaults.toml](storewolf/defaults.toml), which defines our configuration files and services +* The `defaults.toml` files for each [variant](../models), which define our configuration files and services This is a simple startup service that runs `thar-be-settings --all` to write out all of the configuration files that are based on our settings. diff --git a/workspaces/api/apiserver/Cargo.toml b/workspaces/api/apiserver/Cargo.toml index dad2355a001..fa8bea03c33 100644 --- a/workspaces/api/apiserver/Cargo.toml +++ b/workspaces/api/apiserver/Cargo.toml @@ -8,20 +8,16 @@ build = "build.rs" [dependencies] actix-web = { version = "1.0.5", default-features = false, features = ["uds"] } -base64 = "0.10" -lazy_static = "1.2" libc = "0.2" log = "0.4" +models = { path = "../../models" } nix = "0.15.0" percent-encoding = "2.1" -regex = "1.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" simplelog = "0.7" snafu = "0.5" systemd = { version = "0.4.0", default-features = false, features = [], optional = true } -toml = "0.5" -url = "2.1" walkdir = "2.2" [features] @@ -34,3 +30,4 @@ cargo-readme = "3.1" [dev-dependencies] maplit = "1.0" +toml = "0.5" diff --git a/workspaces/api/apiserver/README.md b/workspaces/api/apiserver/README.md index cb5e36217dd..96f50eec067 100644 --- a/workspaces/api/apiserver/README.md +++ b/workspaces/api/apiserver/README.md @@ -34,7 +34,7 @@ Requests are directed by `server::router`. ### Model The API is driven by a data model (similar to a schema) defined in Rust. -See the 'model' module. +See the 'models' workspace. All input is deserialized into model types, and all output is serialized from model types, so we can be more confident that data is in the format we expect. The data model describes system settings, services using those settings, and configuration files used by those services. diff --git a/workspaces/api/apiserver/src/lib.rs b/workspaces/api/apiserver/src/lib.rs index a64ba371f2d..b56e8250b47 100644 --- a/workspaces/api/apiserver/src/lib.rs +++ b/workspaces/api/apiserver/src/lib.rs @@ -31,7 +31,7 @@ Requests are directed by `server::router`. ## Model The API is driven by a data model (similar to a schema) defined in Rust. -See the 'model' module. +See the 'models' workspace. All input is deserialized into model types, and all output is serialized from model types, so we can be more confident that data is in the format we expect. The data model describes system settings, services using those settings, and configuration files used by those services. @@ -78,8 +78,6 @@ See `../../apiclient/README.md` for client examples. extern crate log; pub mod datastore; -pub mod model; -pub mod modeled_types; pub mod server; pub use server::serve; diff --git a/workspaces/api/apiserver/src/server/controller.rs b/workspaces/api/apiserver/src/server/controller.rs index cd6762f595b..960d5d0c186 100644 --- a/workspaces/api/apiserver/src/server/controller.rs +++ b/workspaces/api/apiserver/src/server/controller.rs @@ -12,8 +12,8 @@ use crate::datastore::serialization::to_pairs; use crate::datastore::{ deserialize_scalar, Committed, DataStore, Key, KeyType, ScalarError, Value, }; -use crate::model::{ConfigurationFiles, Services, Settings}; use crate::server::error::{self, Result}; +use model::{ConfigurationFiles, Services, Settings}; /// Build a Settings based on pending data in the datastore; the Settings will be empty if there /// are no pending settings. @@ -339,8 +339,8 @@ mod test { use super::*; use crate::datastore::memory::MemoryDataStore; use crate::datastore::{Committed, DataStore, Key, KeyType}; - use crate::model::Service; use maplit::{hashmap, hashset}; + use model::Service; use std::convert::TryInto; #[test] diff --git a/workspaces/api/apiserver/src/server/mod.rs b/workspaces/api/apiserver/src/server/mod.rs index a28c13818fc..84b5c92d653 100644 --- a/workspaces/api/apiserver/src/server/mod.rs +++ b/workspaces/api/apiserver/src/server/mod.rs @@ -12,8 +12,8 @@ use std::path::Path; use std::sync; use crate::datastore::{Committed, FilesystemDataStore, Key, Value}; -use crate::model::{ConfigurationFiles, Services, Settings}; use error::Result; +use model::{ConfigurationFiles, Services, Settings}; use nix::unistd::{chown, Gid}; use std::fs::set_permissions; @@ -122,10 +122,10 @@ where fn get_settings( query: web::Query>, data: web::Data, -) -> Result { +) -> Result { let datastore = data.ds.read().ok().context(error::DataStoreLock)?; - if let Some(keys_str) = query.get("keys") { + let settings = if let Some(keys_str) = query.get("keys") { let keys = comma_separated("keys", keys_str)?; controller::get_settings_keys(&*datastore, &keys, Committed::Live) } else if let Some(prefix_str) = query.get("prefix") { @@ -136,7 +136,9 @@ fn get_settings( controller::get_settings_prefix(&*datastore, prefix_str, Committed::Live) } else { controller::get_settings(&*datastore, Committed::Live) - } + }?; + + Ok(SettingsResponse(settings)) } /// Apply the requested settings to the pending data store @@ -150,9 +152,10 @@ fn patch_settings( } /// Return any settings that have been received but not committed -fn get_pending_settings(data: web::Data) -> Result { +fn get_pending_settings(data: web::Data) -> Result { let datastore = data.ds.read().ok().context(error::DataStoreLock)?; - controller::get_pending_settings(&*datastore) + let settings = controller::get_pending_settings(&*datastore)?; + Ok(SettingsResponse(settings)) } /// Delete any settings that have been received but not committed @@ -338,8 +341,9 @@ macro_rules! impl_responder_for { ) } -// This lets us respond from our handler methods with a Settings (or Result) -impl_responder_for!(Settings, self, self); +/// This lets us respond from our handler methods with a Settings (or Result) +struct SettingsResponse(Settings); +impl_responder_for!(SettingsResponse, self, self.0); /// This lets us respond from our handler methods with a HashMap (or Result) for metadata struct MetadataResponse(HashMap); diff --git a/workspaces/api/host-containers/Cargo.toml b/workspaces/api/host-containers/Cargo.toml index 6dae489c81d..1ebb77f7e01 100644 --- a/workspaces/api/host-containers/Cargo.toml +++ b/workspaces/api/host-containers/Cargo.toml @@ -11,6 +11,7 @@ apiclient = { path = "../apiclient" } apiserver = { path = "../apiserver" } http = "0.1" log = "0.4" +models = { path = "../../models" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" simplelog = "0.7" diff --git a/workspaces/api/host-containers/src/main.rs b/workspaces/api/host-containers/src/main.rs index ecd8e9f0893..5430c9e12ee 100644 --- a/workspaces/api/host-containers/src/main.rs +++ b/workspaces/api/host-containers/src/main.rs @@ -22,8 +22,7 @@ use std::path::{Path, PathBuf}; use std::process::{self, Command}; use std::str::FromStr; -use apiserver::model; -use apiserver::modeled_types::Identifier; +use model::modeled_types::Identifier; // FIXME Get from configuration in the future const DEFAULT_API_SOCKET: &str = "/run/api.sock"; diff --git a/workspaces/api/servicedog/Cargo.toml b/workspaces/api/servicedog/Cargo.toml index d62bc4d9564..dd07bf8a91a 100644 --- a/workspaces/api/servicedog/Cargo.toml +++ b/workspaces/api/servicedog/Cargo.toml @@ -11,6 +11,7 @@ apiclient = { path = "../apiclient" } apiserver = { path = "../apiserver" } http = "0.1" log = "0.4" +models = { path = "../../models" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" simplelog = "0.7" diff --git a/workspaces/api/servicedog/src/main.rs b/workspaces/api/servicedog/src/main.rs index 0e750b2f858..6effd5e846b 100644 --- a/workspaces/api/servicedog/src/main.rs +++ b/workspaces/api/servicedog/src/main.rs @@ -27,7 +27,6 @@ use std::str::FromStr; use apiserver::datastore::serialization::to_pairs_with_prefix; use apiserver::datastore::{Key, KeyType}; -use apiserver::model; // FIXME Get from configuration in the future const DEFAULT_API_SOCKET: &str = "/run/api.sock"; diff --git a/workspaces/api/storewolf/Cargo.toml b/workspaces/api/storewolf/Cargo.toml index e4e5adf4929..ea989b24db7 100644 --- a/workspaces/api/storewolf/Cargo.toml +++ b/workspaces/api/storewolf/Cargo.toml @@ -10,6 +10,7 @@ build = "build.rs" apiserver = { path = "../apiserver" } data_store_version = { path = "../data_store_version" } log = "0.4" +models = { path = "../../models" } rand = { version = "0.7", default-features = false, features = ["std"] } simplelog = "0.7" snafu = "0.5" diff --git a/workspaces/api/storewolf/src/main.rs b/workspaces/api/storewolf/src/main.rs index 44aedeb6461..ce7dc875ff9 100644 --- a/workspaces/api/storewolf/src/main.rs +++ b/workspaces/api/storewolf/src/main.rs @@ -25,9 +25,8 @@ use std::{env, fs, process}; use apiserver::datastore::key::{Key, KeyType}; use apiserver::datastore::serialization::{to_pairs, to_pairs_with_prefix}; use apiserver::datastore::{self, DataStore, FilesystemDataStore, ScalarError}; -use apiserver::model; -use apiserver::modeled_types::SingleLineString; use data_store_version::Version; +use model::modeled_types::SingleLineString; // FIXME Get these from configuration in the future const DATASTORE_VERSION_FILE: &str = "/usr/share/thar/data-store-version"; @@ -38,8 +37,8 @@ mod error { use apiserver::datastore::key::KeyType; use apiserver::datastore::{self, serialization, ScalarError}; - use apiserver::modeled_types::error::Error as ModeledTypesError; use data_store_version::error::Error as DataStoreVersionError; + use model::modeled_types::error::Error as ModeledTypesError; use snafu::Snafu; /// Potential errors during execution @@ -311,7 +310,7 @@ fn populate_default_datastore>( } // Read and parse defaults - let defaults_str = include_str!("../defaults.toml"); + let defaults_str = include_str!("../../../models/current/src/defaults.toml"); let mut defaults_val: toml::Value = toml::from_str(defaults_str).context(error::DefaultsFormatting)?; diff --git a/workspaces/api/sundog/Cargo.toml b/workspaces/api/sundog/Cargo.toml index bc751097483..2b3aac78b51 100644 --- a/workspaces/api/sundog/Cargo.toml +++ b/workspaces/api/sundog/Cargo.toml @@ -11,6 +11,7 @@ apiclient = { path = "../apiclient" } apiserver = { path = "../apiserver" } http = "0.1" log = "0.4" +models = { path = "../../models" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" simplelog = "0.7" diff --git a/workspaces/api/sundog/src/main.rs b/workspaces/api/sundog/src/main.rs index e622eb1e3f5..51e8b2c822c 100644 --- a/workspaces/api/sundog/src/main.rs +++ b/workspaces/api/sundog/src/main.rs @@ -22,7 +22,6 @@ use std::str::{self, FromStr}; use apiserver::datastore::serialization::to_pairs_with_prefix; use apiserver::datastore::{self, deserialization, Key, KeyType}; -use apiserver::model; // FIXME Get from configuration in the future const DEFAULT_API_SOCKET: &str = "/run/api.sock"; diff --git a/workspaces/api/thar-be-settings/Cargo.toml b/workspaces/api/thar-be-settings/Cargo.toml index 9e464ec2d51..0dddb3a3208 100644 --- a/workspaces/api/thar-be-settings/Cargo.toml +++ b/workspaces/api/thar-be-settings/Cargo.toml @@ -14,6 +14,7 @@ handlebars = "2.0" http = "0.1" itertools = "0.8" log = "0.4" +models = { path = "../../models" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" simplelog = "0.7" diff --git a/workspaces/api/thar-be-settings/src/config.rs b/workspaces/api/thar-be-settings/src/config.rs index da4f781fa61..df7c8bf4bb0 100644 --- a/workspaces/api/thar-be-settings/src/config.rs +++ b/workspaces/api/thar-be-settings/src/config.rs @@ -9,8 +9,6 @@ use itertools::join; use crate::client; use crate::{error, Result}; -use apiserver::model; - /// Query the API for ConfigurationFile data #[allow(clippy::implicit_hasher)] pub fn get_affected_config_files

( diff --git a/workspaces/api/thar-be-settings/src/service.rs b/workspaces/api/thar-be-settings/src/service.rs index a4b0da5f681..c8c50dc4485 100644 --- a/workspaces/api/thar-be-settings/src/service.rs +++ b/workspaces/api/thar-be-settings/src/service.rs @@ -8,7 +8,6 @@ use itertools::join; use crate::client; use crate::{error, Result}; -use apiserver::model; /// Wrapper for the multiple functions needed to go from /// a list of changed settings to a Services map diff --git a/workspaces/api/thar-be-settings/src/settings.rs b/workspaces/api/thar-be-settings/src/settings.rs index 92974949831..f05af3fbb50 100644 --- a/workspaces/api/thar-be-settings/src/settings.rs +++ b/workspaces/api/thar-be-settings/src/settings.rs @@ -3,8 +3,6 @@ use std::path::Path; use crate::client; use crate::Result; -use apiserver::model; - /// Using the template registry, gather all keys and request /// their values from the API pub fn get_settings_from_template

( diff --git a/workspaces/api/thar-be-settings/src/template.rs b/workspaces/api/thar-be-settings/src/template.rs index bf3e63bbaab..7f650dceceb 100644 --- a/workspaces/api/thar-be-settings/src/template.rs +++ b/workspaces/api/thar-be-settings/src/template.rs @@ -3,8 +3,6 @@ use snafu::ResultExt; use crate::{error, helpers, Result}; -use apiserver::model; - /// Build the template registry using the ConfigFile structs /// and let handlebars parse the templates pub fn build_template_registry( diff --git a/workspaces/deny.toml b/workspaces/deny.toml index 0cfc787fab9..20cb0ffde9d 100644 --- a/workspaces/deny.toml +++ b/workspaces/deny.toml @@ -39,6 +39,7 @@ skip = [ { name = "laika", licenses = [] }, { name = "migration-helpers", licenses = [] }, { name = "migrator", licenses = [] }, + { name = "models", licenses = [] }, { name = "moondog", licenses = [] }, { name = "netdog", licenses = [] }, { name = "pluto", licenses = [] }, diff --git a/workspaces/models/.gitignore b/workspaces/models/.gitignore new file mode 100644 index 00000000000..0136e2709b0 --- /dev/null +++ b/workspaces/models/.gitignore @@ -0,0 +1 @@ +/current/src diff --git a/workspaces/models/Cargo.toml b/workspaces/models/Cargo.toml new file mode 100644 index 00000000000..dda4c6eec97 --- /dev/null +++ b/workspaces/models/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "models" +version = "0.1.0" +authors = ["Tom Kirchner "] +edition = "2018" + +[dependencies] +base64 = "0.10" +lazy_static = "1.2" +regex = "1.1" +serde = { version = "1.0", features = ["derive"] } +snafu = "0.5" +toml = "0.5" +url = "2.1" + +[lib] +name = "model" +path = "current/src/model.rs" diff --git a/workspaces/models/README.md b/workspaces/models/README.md new file mode 100644 index 00000000000..d6b56a0f9f5 --- /dev/null +++ b/workspaces/models/README.md @@ -0,0 +1,31 @@ +# API models + +Thar has different variants supporting different features and use cases. +Each variant has its own set of software, and therefore needs its own configuration. +We support having an API model for each variant to support these different configurations. + +## aws-k8s: Kubernetes + +* [Model](aws-k8s/lib.rs) +* [Defaults](aws-k8s/defaults.toml) + +## aws-dev: Development build + +* [Model](aws-dev/lib.rs) +* [Defaults](aws-dev/defaults.toml) + +# This directory + +We use `build.rs` to symlink the proper API model source code for Cargo to build. +We determine the "proper" model by using the `VARIANT` environment variable. + +If a developer is doing a local `cargo build`, they need to set `VARIANT`. + +When building with the Thar build system, `VARIANT` is based on `BUILDSYS_VARIANT` from the top-level `Makefile.toml`, which can be overridden on the command line with `cargo make -e BUILDSYS_VARIANT=bla`. + +Note: when building with the build system, we can't create the symlink in the source directory during a build - the directories are owned by `root`, but we're `builder`. +We can't use a read/write bind mount with current Docker syntax. +To get around this, in the top-level `Dockerfile`, we mount a "cache" directory at `current` that we can modify. +We set Cargo (via `Cargo.toml`) to look for the source at `current/src`, rather than the default `src`. + +Note: all models share the same `Cargo.toml`. diff --git a/workspaces/models/aws-dev/defaults.toml b/workspaces/models/aws-dev/defaults.toml new file mode 100644 index 00000000000..87d04ba5bcc --- /dev/null +++ b/workspaces/models/aws-dev/defaults.toml @@ -0,0 +1,70 @@ +# OS-level defaults. +# Should match the structures in src/model.rs. + +[settings] +timezone = "America/Los_Angeles" +hostname = "localhost" + +[settings.updates] +metadata-base-url = "https://d25d9m6x9pxh9h.cloudfront.net/45efedef4afe/metadata/" +target-base-url = "https://d25d9m6x9pxh9h.cloudfront.net/45efedef4afe/targets/" + +[services.hostname] +configuration-files = ["hostname"] +restart-commands = [] + +[configuration-files.hostname] +path = "/etc/hostname" +template-path = "/usr/share/templates/hostname" + +[metadata.settings.hostname] +affected-services = ["hostname"] + +# Updog. + +[services.updog] +configuration-files = ["updog-toml"] +restart-commands = [] + +[configuration-files.updog-toml] +path = "/etc/updog.toml" +template-path = "/usr/share/templates/updog-toml" + +[metadata.settings.updates] +affected-services = ["updog"] +seed.setting-generator = "bork seed" + +# HostContainers + +[settings.host-containers.admin] +enabled = false +source = "328549459982.dkr.ecr.us-west-2.amazonaws.com/thar-admin:v0.2" +superpowered = true + +[settings.host-containers.control] +enabled = true +source = "328549459982.dkr.ecr.us-west-2.amazonaws.com/thar-control:v0.2" +superpowered = false + +[services.host-containers] +configuration-files = [] +restart-commands = ["/usr/bin/host-containers"] + +[metadata.settings.host-containers] +affected-services = ["host-containers"] + +# NTP + +[settings.ntp] +time-servers = ["169.254.169.123", "2.amazon.pool.ntp.org"] + +[services.ntp] +configuration-files = ["chrony-conf"] +restart-commands = ["/bin/systemctl try-reload-or-restart chronyd.service"] + +[configuration-files.chrony-conf] +path = "/etc/chrony.conf" +template-path = "/usr/share/templates/chrony-conf" + +[metadata.settings.ntp] +affected-services = ["chronyd"] diff --git a/workspaces/models/aws-dev/model.rs b/workspaces/models/aws-dev/model.rs new file mode 100644 index 00000000000..fea27bdfb84 --- /dev/null +++ b/workspaces/models/aws-dev/model.rs @@ -0,0 +1,110 @@ +//! The model module is the schema for the data store. +//! +//! The datastore::serialization and datastore::deserialization modules make it easy to map between +//! Rust types and the datastore, and thus, all inputs and outputs are type-checked. + +pub mod modeled_types; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::modeled_types::{Identifier, SingleLineString, Url}; + +///// Primary user-visible settings + +// Note: fields are marked with skip_serializing_if=Option::is_none so that settings GETs don't +// show field=null for everything that isn't set in the relevant group of settings. + +// Note: we have to use 'rename' here because the top-level Settings structure is the only one +// that uses its name in serialization; internal structures use the field name that points to it +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "settings", rename_all = "kebab-case")] +pub struct Settings { + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub updates: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub host_containers: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ntp: Option, +} + +// Updog settings. Taken from userdata. The 'seed' setting is generated +// by the "Bork" settings generator at runtime. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct UpdatesSettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata_base_url: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub target_base_url: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct ContainerImage { + #[serde(skip_serializing_if = "Option::is_none")] + pub source: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub superpowered: Option, +} + +// NTP settings +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct NtpSettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub time_servers: Option>, +} + +///// Internal services + +// Note: Top-level objects that get returned from the API should have a serde "rename" attribute +// matching the struct name, but in kebab-case, e.g. ConfigurationFiles -> "configuration-files". +// This lets it match the datastore name. +// Objects that live inside those top-level objects, e.g. Service lives in Services, should have +// rename="" so they don't add an extra prefix to the datastore path that doesn't actually exist. +// This is important because we have APIs that can return those sub-structures directly. + +pub type Services = HashMap; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "", rename_all = "kebab-case")] +pub struct Service { + pub configuration_files: Vec, + pub restart_commands: Vec, +} + +pub type ConfigurationFiles = HashMap; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "", rename_all = "kebab-case")] +pub struct ConfigurationFile { + pub path: SingleLineString, + pub template_path: SingleLineString, +} + +///// Metadata + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "metadata", rename_all = "kebab-case")] +pub struct Metadata { + pub key: SingleLineString, + pub md: SingleLineString, + pub val: toml::Value, +} diff --git a/workspaces/models/aws-dev/modeled_types.rs b/workspaces/models/aws-dev/modeled_types.rs new file mode 100644 index 00000000000..1930cfe7d0f --- /dev/null +++ b/workspaces/models/aws-dev/modeled_types.rs @@ -0,0 +1,321 @@ +//! This module contains data types that can be used in the model when special input/output +//! (ser/de) behavior is desired. For example, the SingleLineString type can be used for a model field +//! when we don't even want to accept an API call with multiple lines in the input. + +// The pattern in this file is to make a struct and implement TryFrom<&str> with code that does +// necessary checks and returns the struct. Other traits that treat the struct like a string can +// be implemented for you with the string_impls_for macro. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +// Just need serde's Error in scope to get its trait methods +use serde::de::Error as _; +use snafu::ensure; +use std::borrow::Borrow; +use std::convert::TryFrom; +use std::fmt; +use std::ops::Deref; + +pub mod error { + use snafu::Snafu; + + #[derive(Debug, Snafu)] + #[snafu(visibility = "pub(super)")] + pub enum Error { + #[snafu(display("Can't create SingleLineString containing line terminator"))] + StringContainsLineTerminator, + + #[snafu(display("Invalid base64 input: {}", source))] + InvalidBase64 { source: base64::DecodeError }, + + #[snafu(display( + "Identifiers may only contain ASCII alphanumerics plus hyphens, received '{}'", + input + ))] + InvalidIdentifier { input: String }, + + #[snafu(display("Given invalid URL '{}'", input))] + InvalidUrl { input: String }, + + // Some regexes are too big to usefully display in an error. + #[snafu(display("{} given invalid input: {}", thing, input))] + BigPattern { thing: String, input: String }, + + #[snafu(display("Given invalid cluster name '{}': {}", name, msg))] + InvalidClusterName { name: String, msg: String }, + } +} + +/// Helper macro for implementing the common string-like traits for a modeled type. +/// Pass the name of the type, and the name of the type in quotes (to be used in string error +/// messages, etc.). +macro_rules! string_impls_for { + ($for:ident, $for_str:expr) => { + impl TryFrom for $for { + type Error = error::Error; + + fn try_from(input: String) -> Result { + Self::try_from(input.as_ref()) + } + } + + impl<'de> Deserialize<'de> for $for { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let original = String::deserialize(deserializer)?; + Self::try_from(original).map_err(|e| { + D::Error::custom(format!("Unable to deserialize into {}: {}", $for_str, e)) + }) + } + } + + /// We want to serialize the original string back out, not our structure, which is just there to + /// force validation. + impl Serialize for $for { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.inner) + } + } + + impl Deref for $for { + type Target = str; + fn deref(&self) -> &Self::Target { + &self.inner + } + } + + impl Borrow for $for { + fn borrow(&self) -> &String { + &self.inner + } + } + + impl Borrow for $for { + fn borrow(&self) -> &str { + &self.inner + } + } + + impl AsRef for $for { + fn as_ref(&self) -> &str { + &self.inner + } + } + + impl fmt::Display for $for { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.inner) + } + } + + impl From<$for> for String { + fn from(x: $for) -> Self { + x.inner + } + } + }; +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// SingleLineString can only be created by deserializing from a string that contains at most one +/// line. It stores the original form and makes it accessible through standard traits. Its +/// purpose is input validation, for example in cases where you want to accept input for a +/// configuration file and want to ensure a user can't create a new line with extra configuration. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct SingleLineString { + inner: String, +} + +impl TryFrom<&str> for SingleLineString { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + // Rust does not treat all Unicode line terminators as starting a new line, so we check for + // specific characters here, rather than just counting from lines(). + // https://en.wikipedia.org/wiki/Newline#Unicode + let line_terminators = [ + '\n', // newline (0A) + '\r', // carriage return (0D) + '\u{000B}', // vertical tab + '\u{000C}', // form feed + '\u{0085}', // next line + '\u{2028}', // line separator + '\u{2029}', // paragraph separator + ]; + + ensure!( + !input.contains(&line_terminators[..]), + error::StringContainsLineTerminator + ); + + Ok(Self { + inner: input.to_string(), + }) + } +} + +string_impls_for!(SingleLineString, "SingleLineString"); + +#[cfg(test)] +mod test_single_line_string { + use super::SingleLineString; + use std::convert::TryFrom; + + #[test] + fn valid_single_line_string() { + assert!(SingleLineString::try_from("").is_ok()); + assert!(SingleLineString::try_from("hi").is_ok()); + let long_string = std::iter::repeat(" ").take(9999).collect::(); + let json_long_string = format!("{}", &long_string); + assert!(SingleLineString::try_from(json_long_string).is_ok()); + } + + #[test] + fn invalid_single_line_string() { + assert!(SingleLineString::try_from("Hello\nWorld").is_err()); + + assert!(SingleLineString::try_from("\n").is_err()); + assert!(SingleLineString::try_from("\r").is_err()); + assert!(SingleLineString::try_from("\r\n").is_err()); + + assert!(SingleLineString::try_from("\u{000B}").is_err()); // vertical tab + assert!(SingleLineString::try_from("\u{000C}").is_err()); // form feed + assert!(SingleLineString::try_from("\u{0085}").is_err()); // next line + assert!(SingleLineString::try_from("\u{2028}").is_err()); // line separator + assert!(SingleLineString::try_from("\u{2029}").is_err()); + // paragraph separator + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// Identifier can only be created by deserializing from a string that contains +/// ASCII alphanumeric characters, plus hyphens, which we use as our standard word separator +/// character in user-facing identifiers. It stores the original form and makes it accessible +/// through standard traits. Its purpose is to validate input for identifiers like container names +/// that might be used to create files/directories. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct Identifier { + inner: String, +} + +impl TryFrom<&str> for Identifier { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + input + .chars() + .all(|c| (c.is_ascii() && c.is_alphanumeric()) || c == '-'), + error::InvalidIdentifier { input } + ); + Ok(Identifier { + inner: input.to_string(), + }) + } +} + +string_impls_for!(Identifier, "Identifier"); + +#[cfg(test)] +mod test_valid_identifier { + use super::Identifier; + use std::convert::TryFrom; + + #[test] + fn valid_identifier() { + assert!(Identifier::try_from("hello-world").is_ok()); + assert!(Identifier::try_from("helloworld").is_ok()); + assert!(Identifier::try_from("123321hello").is_ok()); + assert!(Identifier::try_from("hello-1234").is_ok()); + assert!(Identifier::try_from("--------").is_ok()); + assert!(Identifier::try_from("11111111").is_ok()); + } + + #[test] + fn invalid_identifier() { + assert!(Identifier::try_from("../").is_err()); + assert!(Identifier::try_from("{}").is_err()); + assert!(Identifier::try_from("hello|World").is_err()); + assert!(Identifier::try_from("hello\nWorld").is_err()); + assert!(Identifier::try_from("hello_world").is_err()); + assert!(Identifier::try_from("タール").is_err()); + assert!(Identifier::try_from("💝").is_err()); + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// Url represents a string that contains a valid URL, according to url::Url, though it also +/// allows URLs without a scheme (e.g. without "http://") because it's common. It stores the +/// original string and makes it accessible through standard traits. Its purpose is to validate +/// input for any field containing a network address. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct Url { + inner: String, +} + +impl TryFrom<&str> for Url { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + if let Ok(_) = input.parse::() { + return Ok(Url { + inner: input.to_string(), + }); + } else { + // It's very common to specify URLs without a scheme, so we add one and see if that + // fixes parsing. + let prefixed = format!("http://{}", input); + if let Ok(_) = prefixed.parse::() { + return Ok(Url { + inner: input.to_string(), + }); + } + } + error::InvalidUrl { input }.fail() + } +} + +string_impls_for!(Url, "Url"); + +#[cfg(test)] +mod test_url { + use super::Url; + use std::convert::TryFrom; + + #[test] + fn good_urls() { + for ok in &[ + "https://example.com/path", + "https://example.com", + "example.com/path", + "example.com", + "ntp://127.0.0.1/path", + "ntp://127.0.0.1", + "127.0.0.1/path", + "127.0.0.1", + "http://localhost/path", + "http://localhost", + "localhost/path", + "localhost", + ] { + Url::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_urls() { + for err in &[ + "how are you", + "weird@", + ] { + Url::try_from(*err).unwrap_err(); + } + } +} diff --git a/workspaces/api/storewolf/defaults.toml b/workspaces/models/aws-k8s/defaults.toml similarity index 100% rename from workspaces/api/storewolf/defaults.toml rename to workspaces/models/aws-k8s/defaults.toml diff --git a/workspaces/api/apiserver/src/model.rs b/workspaces/models/aws-k8s/model.rs similarity index 99% rename from workspaces/api/apiserver/src/model.rs rename to workspaces/models/aws-k8s/model.rs index 7c418e1c979..46e8337c022 100644 --- a/workspaces/api/apiserver/src/model.rs +++ b/workspaces/models/aws-k8s/model.rs @@ -3,6 +3,8 @@ //! The datastore::serialization and datastore::deserialization modules make it easy to map between //! Rust types and the datastore, and thus, all inputs and outputs are type-checked. +pub mod modeled_types; + use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::Ipv4Addr; diff --git a/workspaces/api/apiserver/src/modeled_types.rs b/workspaces/models/aws-k8s/modeled_types.rs similarity index 100% rename from workspaces/api/apiserver/src/modeled_types.rs rename to workspaces/models/aws-k8s/modeled_types.rs diff --git a/workspaces/models/build.rs b/workspaces/models/build.rs new file mode 100644 index 00000000000..68e92524a41 --- /dev/null +++ b/workspaces/models/build.rs @@ -0,0 +1,57 @@ +// The src/ directory is a link to the API model we actually want to build; this build.rs creates +// that symlink based on the VARIANT environment variable, which either comes from the build +// system or the user, if doing a local `cargo build`. +// +// See README.md to understand the symlink setup. + +use std::env; +use std::fs; +use std::io; +use std::os::unix::fs::symlink; +use std::path::Path; +use std::process; + +fn symlink_force(target: P1, link: P2) -> io::Result<()> +where + P1: AsRef, + P2: AsRef, +{ + // Remove link if it already exists + if let Err(e) = fs::remove_file(&link) { + if e.kind() != io::ErrorKind::NotFound { + return Err(e); + } + } + // Link to requested target + symlink(&target, &link) +} + +fn main() { + // The VARIANT variable is originally BUILDSYS_VARIANT, set in the top-level Makefile.toml, + // and is passed through as VARIANT by the top-level Dockerfile. It represents which OS + // variant we're building, and therefore which API model to use. + let var = "VARIANT"; + println!("cargo:rerun-if-env-changed={}", var); + let variant = env::var(var).unwrap_or_else(|_| { + eprintln!("For local builds, you must set the {} environment variable so we know which API model to build against. Valid values are the directories in workspaces/models, for example \"aws-k8s\".", var); + process::exit(1); + }); + + // Point to source directory for requested variant + let link = "current/src"; + let target = format!("../{}", variant); + + // Make sure requested variant exists + // (note: the "../" in `target` is because the link goes into `current/` - we're checking at + // the same level here + if !Path::new(&variant).exists() { + eprintln!("The environment variable {} should refer to a directory under workspaces/models with an API model, but it's set to '{}' which doesn't exist", var, variant); + process::exit(1); + } + + // Create the symlink for the following `cargo build` to use for its source code + symlink_force(&target, link).unwrap_or_else(|e| { + eprintln!("Failed to create symlink at '{}' pointing to '{}' - we need this to support different API models for different variants. Error: {}", link, target, e); + process::exit(1); + }); +} diff --git a/workspaces/models/current/.keep b/workspaces/models/current/.keep new file mode 100644 index 00000000000..596f9f2c962 --- /dev/null +++ b/workspaces/models/current/.keep @@ -0,0 +1,3 @@ +We want git to track this directory so that it exists for end-user builds. +It's needed by build.rs and the symlinks described in README.md. +It's clearer what's going on if the directory exists already, rather than build.rs creating it.