From 5cf7da6195e8d04c7bffb29590c644ae033c3f7f Mon Sep 17 00:00:00 2001 From: Tom Kirchner Date: Mon, 16 Dec 2019 15:27:03 -0800 Subject: [PATCH] Refactor models to reduce duplication To be able to have `models/lib.rs` with shared code, we needed to stop symlinking the entire `src` directory and instead treat the variant-specific code as Rust modules, symlinking the correct one to `variant::current` so that `lib.rs` can re-export `Settings`. (We also have to symlink in `variant/mod.rs` so Rust knows it's part of the module hierarchy.) Now, each variant defines a `Settings` structure using common sub-structures. Those sub-structures live in the main `lib.rs`, and the modeled types are now in a shared `modeled_types` module. No changes were made to the code, just its organization. Documentation was updated to reflect these changes, including using cargo-readme to generate README.md now that there's a single source. --- Dockerfile | 2 +- workspaces/Cargo.lock | 1 + workspaces/api/storewolf/src/main.rs | 2 +- workspaces/models/.gitignore | 3 +- workspaces/models/Cargo.toml | 9 +- workspaces/models/README.md | 40 +- workspaces/models/README.tpl | 9 + workspaces/models/aws-dev/model.rs | 110 --- workspaces/models/aws-k8s/modeled_types.rs | 706 ------------------ workspaces/models/build.rs | 67 +- .../models/{ => src}/aws-dev/defaults.toml | 2 +- workspaces/models/src/aws-dev/mod.rs | 26 + .../models/{ => src}/aws-k8s/defaults.toml | 2 +- workspaces/models/src/aws-k8s/mod.rs | 29 + .../models/{aws-k8s/model.rs => src/lib.rs} | 89 ++- .../models/src/modeled_types/kubernetes.rs | 339 +++++++++ workspaces/models/src/modeled_types/mod.rs | 127 ++++ .../modeled_types/shared.rs} | 139 +--- .../models/{current => src/variant}/.keep | 0 workspaces/models/src/variant_mod.rs | 4 + 20 files changed, 724 insertions(+), 982 deletions(-) create mode 100644 workspaces/models/README.tpl delete mode 100644 workspaces/models/aws-dev/model.rs delete mode 100644 workspaces/models/aws-k8s/modeled_types.rs rename workspaces/models/{ => src}/aws-dev/defaults.toml (97%) create mode 100644 workspaces/models/src/aws-dev/mod.rs rename workspaces/models/{ => src}/aws-k8s/defaults.toml (98%) create mode 100644 workspaces/models/src/aws-k8s/mod.rs rename workspaces/models/{aws-k8s/model.rs => src/lib.rs} (58%) create mode 100644 workspaces/models/src/modeled_types/kubernetes.rs create mode 100644 workspaces/models/src/modeled_types/mod.rs rename workspaces/models/{aws-dev/modeled_types.rs => src/modeled_types/shared.rs} (65%) rename workspaces/models/{current => src/variant}/.keep (100%) create mode 100644 workspaces/models/src/variant_mod.rs diff --git a/Dockerfile b/Dockerfile index 4fa40bff0d6..55d5da49a93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,7 @@ RUN --mount=target=/host \ USER builder RUN --mount=source=.cargo,target=/home/builder/.cargo \ --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \ - --mount=type=cache,target=/home/builder/rpmbuild/BUILD/workspaces/models/current,from=variantcache,source=/variantcache \ + --mount=type=cache,target=/home/builder/rpmbuild/BUILD/workspaces/models/src/variant,from=variantcache,source=/variantcache \ --mount=source=workspaces,target=/home/builder/rpmbuild/BUILD/workspaces \ rpmbuild -ba --clean rpmbuild/SPECS/${PACKAGE}.spec diff --git a/workspaces/Cargo.lock b/workspaces/Cargo.lock index 75ac3523314..3cd4f35d393 100644 --- a/workspaces/Cargo.lock +++ b/workspaces/Cargo.lock @@ -1417,6 +1417,7 @@ name = "models" version = "0.1.0" dependencies = [ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cargo-readme 3.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/workspaces/api/storewolf/src/main.rs b/workspaces/api/storewolf/src/main.rs index ce7dc875ff9..aefe7ed1136 100644 --- a/workspaces/api/storewolf/src/main.rs +++ b/workspaces/api/storewolf/src/main.rs @@ -310,7 +310,7 @@ fn populate_default_datastore>( } // Read and parse defaults - let defaults_str = include_str!("../../../models/current/src/defaults.toml"); + let defaults_str = include_str!("../../../models/src/variant/current/defaults.toml"); let mut defaults_val: toml::Value = toml::from_str(defaults_str).context(error::DefaultsFormatting)?; diff --git a/workspaces/models/.gitignore b/workspaces/models/.gitignore index 0136e2709b0..b539ea1d8bc 100644 --- a/workspaces/models/.gitignore +++ b/workspaces/models/.gitignore @@ -1 +1,2 @@ -/current/src +/src/variant/current +/src/variant/mod.rs diff --git a/workspaces/models/Cargo.toml b/workspaces/models/Cargo.toml index dda4c6eec97..d7e0a2c9807 100644 --- a/workspaces/models/Cargo.toml +++ b/workspaces/models/Cargo.toml @@ -3,6 +3,8 @@ name = "models" version = "0.1.0" authors = ["Tom Kirchner "] edition = "2018" +publish = false +build = "build.rs" [dependencies] base64 = "0.10" @@ -13,6 +15,11 @@ snafu = "0.5" toml = "0.5" url = "2.1" +[build-dependencies] +cargo-readme = "3.1" + [lib] +# We're picking the current *model* with build.rs, so users shouldn't think +# about importing *models* (plural), just the one current model. name = "model" -path = "current/src/model.rs" +path = "src/lib.rs" diff --git a/workspaces/models/README.md b/workspaces/models/README.md index d6b56a0f9f5..a53edf685a4 100644 --- a/workspaces/models/README.md +++ b/workspaces/models/README.md @@ -1,20 +1,32 @@ -# API models +# models + +Current version: 0.1.0 + +## API models Thar has different variants supporting different features and use cases. Each variant has its own set of software, and therefore needs its own configuration. We support having an API model for each variant to support these different configurations. -## aws-k8s: Kubernetes +Each model defines a top-level `Settings` structure. +It can use pre-defined structures inside, or custom ones as needed. + +This `Settings` essentially becomes the schema for the variant's data store. +`apiserver::datastore` offers serialization and deserialization modules that make it easy to map between Rust types and the data store, and thus, all inputs and outputs are type-checked. + +At the field level, standard Rust types can be used, or ["modeled types"](src/modeled_types) that add input validation. -* [Model](aws-k8s/lib.rs) -* [Defaults](aws-k8s/defaults.toml) +### aws-k8s: Kubernetes -## aws-dev: Development build +* [Model](src/aws-k8s/mod.rs) +* [Defaults](src/aws-k8s/defaults.toml) -* [Model](aws-dev/lib.rs) -* [Defaults](aws-dev/defaults.toml) +### aws-dev: Development build -# This directory +* [Model](src/aws-dev/mod.rs) +* [Defaults](src/aws-dev/defaults.toml) + +## This directory We use `build.rs` to symlink the proper API model source code for Cargo to build. We determine the "proper" model by using the `VARIANT` environment variable. @@ -25,7 +37,15 @@ When building with the Thar build system, `VARIANT` is based on `BUILDSYS_VARIAN Note: when building with the build system, we can't create the symlink in the source directory during a build - the directories are owned by `root`, but we're `builder`. We can't use a read/write bind mount with current Docker syntax. -To get around this, in the top-level `Dockerfile`, we mount a "cache" directory at `current` that we can modify. -We set Cargo (via `Cargo.toml`) to look for the source at `current/src`, rather than the default `src`. +To get around this, in the top-level `Dockerfile`, we mount a "cache" directory at `src/variant` that we can modify, and create a `current` symlink inside. +The code in `src/lib.rs` then imports the requested model using `variant/current`. + +Note: for the same reason, we symlink `variant/mod.rs` to `variant_mod.rs`. +Rust needs a `mod.rs` file to understand that a directory is part of the module structure, so we have to have `variant/mod.rs`. +`variant/` is the cache mount that starts empty, so we have to store the file elsewhere and link it in. Note: all models share the same `Cargo.toml`. + +## Colophon + +This text was generated from `README.tpl` using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/lib.rs`. \ No newline at end of file diff --git a/workspaces/models/README.tpl b/workspaces/models/README.tpl new file mode 100644 index 00000000000..91fb62910c8 --- /dev/null +++ b/workspaces/models/README.tpl @@ -0,0 +1,9 @@ +# {{crate}} + +Current version: {{version}} + +{{readme}} + +## Colophon + +This text was generated from `README.tpl` using [cargo-readme](https://crates.io/crates/cargo-readme), and includes the rustdoc from `src/lib.rs`. diff --git a/workspaces/models/aws-dev/model.rs b/workspaces/models/aws-dev/model.rs deleted file mode 100644 index fea27bdfb84..00000000000 --- a/workspaces/models/aws-dev/model.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! The model module is the schema for the data store. -//! -//! The datastore::serialization and datastore::deserialization modules make it easy to map between -//! Rust types and the datastore, and thus, all inputs and outputs are type-checked. - -pub mod modeled_types; - -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -use crate::modeled_types::{Identifier, SingleLineString, Url}; - -///// Primary user-visible settings - -// Note: fields are marked with skip_serializing_if=Option::is_none so that settings GETs don't -// show field=null for everything that isn't set in the relevant group of settings. - -// Note: we have to use 'rename' here because the top-level Settings structure is the only one -// that uses its name in serialization; internal structures use the field name that points to it -#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename = "settings", rename_all = "kebab-case")] -pub struct Settings { - #[serde(skip_serializing_if = "Option::is_none")] - pub timezone: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub hostname: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub updates: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub host_containers: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub ntp: Option, -} - -// Updog settings. Taken from userdata. The 'seed' setting is generated -// by the "Bork" settings generator at runtime. -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub struct UpdatesSettings { - #[serde(skip_serializing_if = "Option::is_none")] - pub metadata_base_url: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub target_base_url: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub seed: Option, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub struct ContainerImage { - #[serde(skip_serializing_if = "Option::is_none")] - pub source: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub superpowered: Option, -} - -// NTP settings -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub struct NtpSettings { - #[serde(skip_serializing_if = "Option::is_none")] - pub time_servers: Option>, -} - -///// Internal services - -// Note: Top-level objects that get returned from the API should have a serde "rename" attribute -// matching the struct name, but in kebab-case, e.g. ConfigurationFiles -> "configuration-files". -// This lets it match the datastore name. -// Objects that live inside those top-level objects, e.g. Service lives in Services, should have -// rename="" so they don't add an extra prefix to the datastore path that doesn't actually exist. -// This is important because we have APIs that can return those sub-structures directly. - -pub type Services = HashMap; - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename = "", rename_all = "kebab-case")] -pub struct Service { - pub configuration_files: Vec, - pub restart_commands: Vec, -} - -pub type ConfigurationFiles = HashMap; - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename = "", rename_all = "kebab-case")] -pub struct ConfigurationFile { - pub path: SingleLineString, - pub template_path: SingleLineString, -} - -///// Metadata - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename = "metadata", rename_all = "kebab-case")] -pub struct Metadata { - pub key: SingleLineString, - pub md: SingleLineString, - pub val: toml::Value, -} diff --git a/workspaces/models/aws-k8s/modeled_types.rs b/workspaces/models/aws-k8s/modeled_types.rs deleted file mode 100644 index 3b6d05f600e..00000000000 --- a/workspaces/models/aws-k8s/modeled_types.rs +++ /dev/null @@ -1,706 +0,0 @@ -//! This module contains data types that can be used in the model when special input/output -//! (ser/de) behavior is desired. For example, the ValidBase64 type can be used for a model field -//! when we don't even want to accept an API call with invalid base64 data. - -// The pattern in this file is to make a struct and implement TryFrom<&str> with code that does -// necessary checks and returns the struct. Other traits that treat the struct like a string can -// be implemented for you with the string_impls_for macro. - -use lazy_static::lazy_static; -use regex::Regex; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -// Just need serde's Error in scope to get its trait methods -use serde::de::Error as _; -use snafu::{ensure, ResultExt}; -use std::borrow::Borrow; -use std::convert::TryFrom; -use std::fmt; -use std::ops::Deref; - -pub mod error { - use regex::Regex; - use snafu::Snafu; - - #[derive(Debug, Snafu)] - #[snafu(visibility = "pub(super)")] - pub enum Error { - #[snafu(display("Can't create SingleLineString containing line terminator"))] - StringContainsLineTerminator, - - #[snafu(display("Invalid base64 input: {}", source))] - InvalidBase64 { source: base64::DecodeError }, - - #[snafu(display( - "Identifiers may only contain ASCII alphanumerics plus hyphens, received '{}'", - input - ))] - InvalidIdentifier { input: String }, - - #[snafu(display("Given invalid URL '{}'", input))] - InvalidUrl { input: String }, - - #[snafu(display("{} must match '{}', given: {}", thing, pattern, input))] - Pattern { - thing: String, - pattern: Regex, - input: String, - }, - - // Some regexes are too big to usefully display in an error. - #[snafu(display("{} given invalid input: {}", thing, input))] - BigPattern { thing: String, input: String }, - - #[snafu(display("Given invalid cluster name '{}': {}", name, msg))] - InvalidClusterName { name: String, msg: String }, - } -} - -/// Helper macro for implementing the common string-like traits for a modeled type. -/// Pass the name of the type, and the name of the type in quotes (to be used in string error -/// messages, etc.). -macro_rules! string_impls_for { - ($for:ident, $for_str:expr) => { - impl TryFrom for $for { - type Error = error::Error; - - fn try_from(input: String) -> Result { - Self::try_from(input.as_ref()) - } - } - - impl<'de> Deserialize<'de> for $for { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let original = String::deserialize(deserializer)?; - Self::try_from(original).map_err(|e| { - D::Error::custom(format!("Unable to deserialize into {}: {}", $for_str, e)) - }) - } - } - - /// We want to serialize the original string back out, not our structure, which is just there to - /// force validation. - impl Serialize for $for { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.inner) - } - } - - impl Deref for $for { - type Target = str; - fn deref(&self) -> &Self::Target { - &self.inner - } - } - - impl Borrow for $for { - fn borrow(&self) -> &String { - &self.inner - } - } - - impl Borrow for $for { - fn borrow(&self) -> &str { - &self.inner - } - } - - impl AsRef for $for { - fn as_ref(&self) -> &str { - &self.inner - } - } - - impl fmt::Display for $for { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.inner) - } - } - - impl From<$for> for String { - fn from(x: $for) -> Self { - x.inner - } - } - }; -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// ValidBase64 can only be created by deserializing from valid base64 text. It stores the -/// original text, not the decoded form. Its purpose is input validation, namely being used as a -/// field in a model structure so that you don't even accept a request with a field that has -/// invalid base64. -// Note: we use the default base64::STANDARD config which uses/allows "=" padding. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct ValidBase64 { - inner: String, -} - -/// Validate base64 format before we accept the input. -impl TryFrom<&str> for ValidBase64 { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - base64::decode(&input).context(error::InvalidBase64)?; - Ok(ValidBase64 { - inner: input.to_string(), - }) - } -} - -string_impls_for!(ValidBase64, "ValidBase64"); - -#[cfg(test)] -mod test_valid_base64 { - use super::ValidBase64; - use std::convert::TryFrom; - - #[test] - fn valid_base64() { - let v = ValidBase64::try_from("aGk=").unwrap(); - let decoded_bytes = base64::decode(v.as_ref()).unwrap(); - let decoded = std::str::from_utf8(&decoded_bytes).unwrap(); - assert_eq!(decoded, "hi"); - } - - #[test] - fn invalid_base64() { - assert!(ValidBase64::try_from("invalid base64").is_err()); - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// SingleLineString can only be created by deserializing from a string that contains at most one -/// line. It stores the original form and makes it accessible through standard traits. Its -/// purpose is input validation, for example in cases where you want to accept input for a -/// configuration file and want to ensure a user can't create a new line with extra configuration. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct SingleLineString { - inner: String, -} - -impl TryFrom<&str> for SingleLineString { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - // Rust does not treat all Unicode line terminators as starting a new line, so we check for - // specific characters here, rather than just counting from lines(). - // https://en.wikipedia.org/wiki/Newline#Unicode - let line_terminators = [ - '\n', // newline (0A) - '\r', // carriage return (0D) - '\u{000B}', // vertical tab - '\u{000C}', // form feed - '\u{0085}', // next line - '\u{2028}', // line separator - '\u{2029}', // paragraph separator - ]; - - ensure!( - !input.contains(&line_terminators[..]), - error::StringContainsLineTerminator - ); - - Ok(Self { - inner: input.to_string(), - }) - } -} - -string_impls_for!(SingleLineString, "SingleLineString"); - -#[cfg(test)] -mod test_single_line_string { - use super::SingleLineString; - use std::convert::TryFrom; - - #[test] - fn valid_single_line_string() { - assert!(SingleLineString::try_from("").is_ok()); - assert!(SingleLineString::try_from("hi").is_ok()); - let long_string = std::iter::repeat(" ").take(9999).collect::(); - let json_long_string = format!("{}", &long_string); - assert!(SingleLineString::try_from(json_long_string).is_ok()); - } - - #[test] - fn invalid_single_line_string() { - assert!(SingleLineString::try_from("Hello\nWorld").is_err()); - - assert!(SingleLineString::try_from("\n").is_err()); - assert!(SingleLineString::try_from("\r").is_err()); - assert!(SingleLineString::try_from("\r\n").is_err()); - - assert!(SingleLineString::try_from("\u{000B}").is_err()); // vertical tab - assert!(SingleLineString::try_from("\u{000C}").is_err()); // form feed - assert!(SingleLineString::try_from("\u{0085}").is_err()); // next line - assert!(SingleLineString::try_from("\u{2028}").is_err()); // line separator - assert!(SingleLineString::try_from("\u{2029}").is_err()); - // paragraph separator - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// Identifier can only be created by deserializing from a string that contains -/// ASCII alphanumeric characters, plus hyphens, which we use as our standard word separator -/// character in user-facing identifiers. It stores the original form and makes it accessible -/// through standard traits. Its purpose is to validate input for identifiers like container names -/// that might be used to create files/directories. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct Identifier { - inner: String, -} - -impl TryFrom<&str> for Identifier { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - input - .chars() - .all(|c| (c.is_ascii() && c.is_alphanumeric()) || c == '-'), - error::InvalidIdentifier { input } - ); - Ok(Identifier { - inner: input.to_string(), - }) - } -} - -string_impls_for!(Identifier, "Identifier"); - -#[cfg(test)] -mod test_valid_identifier { - use super::Identifier; - use std::convert::TryFrom; - - #[test] - fn valid_identifier() { - assert!(Identifier::try_from("hello-world").is_ok()); - assert!(Identifier::try_from("helloworld").is_ok()); - assert!(Identifier::try_from("123321hello").is_ok()); - assert!(Identifier::try_from("hello-1234").is_ok()); - assert!(Identifier::try_from("--------").is_ok()); - assert!(Identifier::try_from("11111111").is_ok()); - } - - #[test] - fn invalid_identifier() { - assert!(Identifier::try_from("../").is_err()); - assert!(Identifier::try_from("{}").is_err()); - assert!(Identifier::try_from("hello|World").is_err()); - assert!(Identifier::try_from("hello\nWorld").is_err()); - assert!(Identifier::try_from("hello_world").is_err()); - assert!(Identifier::try_from("タール").is_err()); - assert!(Identifier::try_from("💝").is_err()); - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// Url represents a string that contains a valid URL, according to url::Url, though it also -/// allows URLs without a scheme (e.g. without "http://") because it's common. It stores the -/// original string and makes it accessible through standard traits. Its purpose is to validate -/// input for any field containing a network address. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct Url { - inner: String, -} - -impl TryFrom<&str> for Url { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - if let Ok(_) = input.parse::() { - return Ok(Url { - inner: input.to_string(), - }); - } else { - // It's very common to specify URLs without a scheme, so we add one and see if that - // fixes parsing. - let prefixed = format!("http://{}", input); - if let Ok(_) = prefixed.parse::() { - return Ok(Url { - inner: input.to_string(), - }); - } - } - error::InvalidUrl { input }.fail() - } -} - -string_impls_for!(Url, "Url"); - -#[cfg(test)] -mod test_url { - use super::Url; - use std::convert::TryFrom; - - #[test] - fn good_urls() { - for ok in &[ - "https://example.com/path", - "https://example.com", - "example.com/path", - "example.com", - "ntp://127.0.0.1/path", - "ntp://127.0.0.1", - "127.0.0.1/path", - "127.0.0.1", - "http://localhost/path", - "http://localhost", - "localhost/path", - "localhost", - ] { - Url::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_urls() { - for err in &[ - "how are you", - "weird@", - ] { - Url::try_from(*err).unwrap_err(); - } - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// KubernetesName represents a string that contains a valid Kubernetes resource name. It stores -/// the original string and makes it accessible through standard traits. -// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesName { - inner: String, -} - -lazy_static! { - pub(crate) static ref KUBERNETES_NAME: Regex = Regex::new(r"^[0-9a-z.-]{1,253}$").unwrap(); -} - -impl TryFrom<&str> for KubernetesName { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - KUBERNETES_NAME.is_match(input), - error::Pattern { - thing: "Kubernetes name", - pattern: KUBERNETES_NAME.clone(), - input - } - ); - Ok(KubernetesName { - inner: input.to_string(), - }) - } -} - -string_impls_for!(KubernetesName, "KubernetesName"); - -#[cfg(test)] -mod test_kubernetes_name { - use super::KubernetesName; - use std::convert::TryFrom; - - #[test] - fn good_names() { - for ok in &["howdy", "42", "18-eighteen."] { - KubernetesName::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_names() { - for err in &["", "HOWDY", "@", "hi/there", &"a".repeat(254)] { - KubernetesName::try_from(*err).unwrap_err(); - } - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// KubernetesLabelKey represents a string that contains a valid Kubernetes label key. It stores -/// the original string and makes it accessible through standard traits. -// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesLabelKey { - inner: String, -} - -lazy_static! { - pub(crate) static ref KUBERNETES_LABEL_KEY: Regex = Regex::new( - r"(?x)^ - ( # optional prefix - [[:alnum:].-]{1,253}/ # DNS label characters followed by slash - )? - [[:alnum:]] # at least one alphanumeric - ( - ([[:alnum:]._-]{0,61})? # more characters allowed in middle - [[:alnum:]] # have to end with alphanumeric - )? - $" - ) - .unwrap(); -} - -impl TryFrom<&str> for KubernetesLabelKey { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - KUBERNETES_LABEL_KEY.is_match(input), - error::BigPattern { - thing: "Kubernetes label key", - input - } - ); - Ok(KubernetesLabelKey { - inner: input.to_string(), - }) - } -} - -string_impls_for!(KubernetesLabelKey, "KubernetesLabelKey"); - -#[cfg(test)] -mod test_kubernetes_label_key { - use super::KubernetesLabelKey; - use std::convert::TryFrom; - - #[test] - fn good_keys() { - for ok in &[ - "no-prefix", - "have.a/prefix", - "more-chars_here.now", - &"a".repeat(63), - &format!("{}/{}", "a".repeat(253), "name"), - ] { - KubernetesLabelKey::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_keys() { - for err in &[ - ".bad", - "bad.", - &"a".repeat(64), - &format!("{}/{}", "a".repeat(254), "name"), - ] { - KubernetesLabelKey::try_from(*err).unwrap_err(); - } - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// KubernetesLabelValue represents a string that contains a valid Kubernetes label value. It -/// stores the original string and makes it accessible through standard traits. -// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesLabelValue { - inner: String, -} - -lazy_static! { - pub(crate) static ref KUBERNETES_LABEL_VALUE: Regex = Regex::new( - r"(?x) - ^$ | # may be empty, or: - ^ - [[:alnum:]] # at least one alphanumeric - ( - ([[:alnum:]._-]{0,61})? # more characters allowed in middle - [[:alnum:]] # have to end with alphanumeric - )? - $ - " - ) - .unwrap(); -} - -impl TryFrom<&str> for KubernetesLabelValue { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - KUBERNETES_LABEL_VALUE.is_match(input), - error::BigPattern { - thing: "Kubernetes label value", - input - } - ); - Ok(KubernetesLabelValue { - inner: input.to_string(), - }) - } -} - -string_impls_for!(KubernetesLabelValue, "KubernetesLabelValue"); - -#[cfg(test)] -mod test_kubernetes_label_value { - use super::KubernetesLabelValue; - use std::convert::TryFrom; - - #[test] - fn good_values() { - for ok in &["", "more-chars_here.now", &"a".repeat(63)] { - KubernetesLabelValue::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_values() { - for err in &[".bad", "bad.", &"a".repeat(64)] { - KubernetesLabelValue::try_from(*err).unwrap_err(); - } - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// KubernetesTaintValue represents a string that contains a valid Kubernetes taint value, which is -/// like a label value, plus a colon, plus an "effect". It stores the original string and makes it -/// accessible through standard traits. -/// -/// Note: Kubelet won't launch if you specify an effect it doesn't know about, but we don't want to -/// gatekeep all possible values, so be careful. -// Note: couldn't find an exact spec for this. Cobbling things together, and guessing a bit as to -// the syntax of the effect. -// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set -// https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesTaintValue { - inner: String, -} - -lazy_static! { - pub(crate) static ref KUBERNETES_TAINT_VALUE: Regex = Regex::new( - r"(?x)^ - [[:alnum:]] # at least one alphanumeric - ( - ([[:alnum:]._-]{0,61})? # more characters allowed in middle - [[:alnum:]] # have to end with alphanumeric - )? - : # separate the label value from the effect - [[:alnum:]]{1,253} # effect - $" - ) - .unwrap(); -} - -impl TryFrom<&str> for KubernetesTaintValue { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - KUBERNETES_TAINT_VALUE.is_match(input), - error::BigPattern { - thing: "Kubernetes taint value", - input - } - ); - Ok(KubernetesTaintValue { - inner: input.to_string(), - }) - } -} - -string_impls_for!(KubernetesTaintValue, "KubernetesTaintValue"); - -#[cfg(test)] -mod test_kubernetes_taint_value { - use super::KubernetesTaintValue; - use std::convert::TryFrom; - - #[test] - fn good_values() { - // All the examples from the docs linked above - for ok in &[ - "value:NoSchedule", - "value:PreferNoSchedule", - "value:NoExecute", - ] { - KubernetesTaintValue::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_values() { - for err in &[".bad", "bad.", &"a".repeat(254), "value:", ":effect"] { - KubernetesTaintValue::try_from(*err).unwrap_err(); - } - } -} - -// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= - -/// KubernetesClusterName represents a string that contains a valid Kubernetes cluster name. It -/// stores the original string and makes it accessible through standard traits. -// Note: I was unable to find the rules for cluster naming. We know they have to fit into label -// values, because of the common cluster-name label, but they also can't be empty. This combines -// those two characteristics into a new type, until we find an explicit syntax. -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub struct KubernetesClusterName { - inner: String, -} - -impl TryFrom<&str> for KubernetesClusterName { - type Error = error::Error; - - fn try_from(input: &str) -> Result { - ensure!( - !input.is_empty(), - error::InvalidClusterName { - name: input, - msg: "must not be empty" - } - ); - ensure!( - KubernetesLabelValue::try_from(input).is_ok(), - error::InvalidClusterName { - name: input, - msg: "cluster names must be valid Kubernetes label values" - } - ); - - Ok(KubernetesClusterName { - inner: input.to_string(), - }) - } -} - -string_impls_for!(KubernetesClusterName, "KubernetesClusterName"); - -#[cfg(test)] -mod test_kubernetes_cluster_name { - use super::KubernetesClusterName; - use std::convert::TryFrom; - - #[test] - fn good_cluster_names() { - for ok in &["more-chars_here.now", &"a".repeat(63)] { - KubernetesClusterName::try_from(*ok).unwrap(); - } - } - - #[test] - fn bad_alues() { - for err in &["", ".bad", "bad.", &"a".repeat(64)] { - KubernetesClusterName::try_from(*err).unwrap_err(); - } - } -} diff --git a/workspaces/models/build.rs b/workspaces/models/build.rs index 68e92524a41..41d2bbd9203 100644 --- a/workspaces/models/build.rs +++ b/workspaces/models/build.rs @@ -1,14 +1,14 @@ -// The src/ directory is a link to the API model we actually want to build; this build.rs creates +// src/variant/current is a link to the API model we actually want to build; this build.rs creates // that symlink based on the VARIANT environment variable, which either comes from the build // system or the user, if doing a local `cargo build`. // // See README.md to understand the symlink setup. use std::env; -use std::fs; -use std::io; +use std::fs::{self, File}; +use std::io::{self, Write}; use std::os::unix::fs::symlink; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process; fn symlink_force(target: P1, link: P2) -> io::Result<()> @@ -26,7 +26,7 @@ where symlink(&target, &link) } -fn main() { +fn link_current_variant() { // The VARIANT variable is originally BUILDSYS_VARIANT, set in the top-level Makefile.toml, // and is passed through as VARIANT by the top-level Dockerfile. It represents which OS // variant we're building, and therefore which API model to use. @@ -37,21 +37,60 @@ fn main() { process::exit(1); }); - // Point to source directory for requested variant - let link = "current/src"; - let target = format!("../{}", variant); + // Point to the source for the requested variant + let variant_link = "src/variant/current"; + let variant_target = format!("../{}", variant); // Make sure requested variant exists - // (note: the "../" in `target` is because the link goes into `current/` - we're checking at - // the same level here - if !Path::new(&variant).exists() { - eprintln!("The environment variable {} should refer to a directory under workspaces/models with an API model, but it's set to '{}' which doesn't exist", var, variant); + let variant_path = format!("src/{}", variant); + if !Path::new(&variant_path).exists() { + eprintln!("The environment variable {} should refer to a directory under workspaces/models/src with an API model, but it's set to '{}' which doesn't exist", var, variant); process::exit(1); } // Create the symlink for the following `cargo build` to use for its source code - symlink_force(&target, link).unwrap_or_else(|e| { - eprintln!("Failed to create symlink at '{}' pointing to '{}' - we need this to support different API models for different variants. Error: {}", link, target, e); + symlink_force(&variant_target, variant_link).unwrap_or_else(|e| { + eprintln!("Failed to create symlink at '{}' pointing to '{}' - we need this to support different API models for different variants. Error: {}", variant_link, variant_target, e); + process::exit(1); + }); + + // Also create the link for mod.rs so Rust can import source from the "current" link + // created above. + let mod_link = "src/variant/mod.rs"; + let mod_target = "../variant_mod.rs"; + symlink_force(&mod_target, mod_link).unwrap_or_else(|e| { + eprintln!("Failed to create symlink at '{}' pointing to '{}' - we need this to build a Rust module structure through the `current` link. Error: {}", mod_link, mod_target, e); process::exit(1); }); } + +fn generate_readme() { + // Check for environment variable "SKIP_README". If it is set, + // skip README generation + if env::var_os("SKIP_README").is_some() { + return; + } + + let mut lib = File::open("src/lib.rs").unwrap(); + let mut template = File::open("README.tpl").unwrap(); + + let content = cargo_readme::generate_readme( + &PathBuf::from("."), // root + &mut lib, // source + Some(&mut template), // template + // The "add x" arguments don't apply when using a template. + true, // add title + false, // add badges + false, // add license + true, // indent headings + ) + .unwrap(); + + let mut readme = File::create("README.md").unwrap(); + readme.write_all(content.as_bytes()).unwrap(); +} + +fn main() { + generate_readme(); + link_current_variant(); +} diff --git a/workspaces/models/aws-dev/defaults.toml b/workspaces/models/src/aws-dev/defaults.toml similarity index 97% rename from workspaces/models/aws-dev/defaults.toml rename to workspaces/models/src/aws-dev/defaults.toml index 3ba1861bae3..4158e7cd21d 100644 --- a/workspaces/models/aws-dev/defaults.toml +++ b/workspaces/models/src/aws-dev/defaults.toml @@ -1,5 +1,5 @@ # OS-level defaults. -# Should match the structures in src/model.rs. +# Should match the structures in the model definition. [settings] timezone = "America/Los_Angeles" diff --git a/workspaces/models/src/aws-dev/mod.rs b/workspaces/models/src/aws-dev/mod.rs new file mode 100644 index 00000000000..dc51b8d09e5 --- /dev/null +++ b/workspaces/models/src/aws-dev/mod.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::modeled_types::{Identifier, SingleLineString}; +use crate::{ContainerImage, NtpSettings, UpdatesSettings}; + +// Note: we have to use 'rename' here because the top-level Settings structure is the only one +// that uses its name in serialization; internal structures use the field name that points to it +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "settings", rename_all = "kebab-case")] +pub struct Settings { + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub updates: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub host_containers: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ntp: Option, +} diff --git a/workspaces/models/aws-k8s/defaults.toml b/workspaces/models/src/aws-k8s/defaults.toml similarity index 98% rename from workspaces/models/aws-k8s/defaults.toml rename to workspaces/models/src/aws-k8s/defaults.toml index 9672784932a..797e3cff142 100644 --- a/workspaces/models/aws-k8s/defaults.toml +++ b/workspaces/models/src/aws-k8s/defaults.toml @@ -1,5 +1,5 @@ # OS-level defaults. -# Should match the structures in src/model.rs. +# Should match the structures in the model definition. [settings] timezone = "America/Los_Angeles" diff --git a/workspaces/models/src/aws-k8s/mod.rs b/workspaces/models/src/aws-k8s/mod.rs new file mode 100644 index 00000000000..098c9a4bdc6 --- /dev/null +++ b/workspaces/models/src/aws-k8s/mod.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::modeled_types::{Identifier, SingleLineString}; +use crate::{ContainerImage, KubernetesSettings, NtpSettings, UpdatesSettings}; + +// Note: we have to use 'rename' here because the top-level Settings structure is the only one +// that uses its name in serialization; internal structures use the field name that poitns to it +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename = "settings", rename_all = "kebab-case")] +pub struct Settings { + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub kubernetes: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub updates: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub host_containers: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub ntp: Option, +} diff --git a/workspaces/models/aws-k8s/model.rs b/workspaces/models/src/lib.rs similarity index 58% rename from workspaces/models/aws-k8s/model.rs rename to workspaces/models/src/lib.rs index 46e8337c022..f98c4ae8d82 100644 --- a/workspaces/models/aws-k8s/model.rs +++ b/workspaces/models/src/lib.rs @@ -1,48 +1,73 @@ -//! The model module is the schema for the data store. -//! -//! The datastore::serialization and datastore::deserialization modules make it easy to map between -//! Rust types and the datastore, and thus, all inputs and outputs are type-checked. +/*! +# API models +Thar has different variants supporting different features and use cases. +Each variant has its own set of software, and therefore needs its own configuration. +We support having an API model for each variant to support these different configurations. + +Each model defines a top-level `Settings` structure. +It can use pre-defined structures inside, or custom ones as needed. + +This `Settings` essentially becomes the schema for the variant's data store. +`apiserver::datastore` offers serialization and deserialization modules that make it easy to map between Rust types and the data store, and thus, all inputs and outputs are type-checked. + +At the field level, standard Rust types can be used, or ["modeled types"](src/modeled_types) that add input validation. + +## aws-k8s: Kubernetes + +* [Model](src/aws-k8s/mod.rs) +* [Defaults](src/aws-k8s/defaults.toml) + +## aws-dev: Development build + +* [Model](src/aws-dev/mod.rs) +* [Defaults](src/aws-dev/defaults.toml) + +# This directory + +We use `build.rs` to symlink the proper API model source code for Cargo to build. +We determine the "proper" model by using the `VARIANT` environment variable. + +If a developer is doing a local `cargo build`, they need to set `VARIANT`. + +When building with the Thar build system, `VARIANT` is based on `BUILDSYS_VARIANT` from the top-level `Makefile.toml`, which can be overridden on the command line with `cargo make -e BUILDSYS_VARIANT=bla`. + +Note: when building with the build system, we can't create the symlink in the source directory during a build - the directories are owned by `root`, but we're `builder`. +We can't use a read/write bind mount with current Docker syntax. +To get around this, in the top-level `Dockerfile`, we mount a "cache" directory at `src/variant` that we can modify, and create a `current` symlink inside. +The code in `src/lib.rs` then imports the requested model using `variant/current`. + +Note: for the same reason, we symlink `variant/mod.rs` to `variant_mod.rs`. +Rust needs a `mod.rs` file to understand that a directory is part of the module structure, so we have to have `variant/mod.rs`. +`variant/` is the cache mount that starts empty, so we have to store the file elsewhere and link it in. + +Note: all models share the same `Cargo.toml`. +*/ + +// "Modeled types" are types with special ser/de behavior used for validation. pub mod modeled_types; +// The "variant" module is just a directory where we symlink in the user's requested build +// variant; each variant defines a top-level Settings structure and we re-export the current one. +mod variant; +pub use variant::Settings; + +// Below, we define common structures used in the API surface; specific variants build a Settings +// structure based on these, and that's what gets exposed via the API. (Specific variants' models +// are in subdirectories and linked into place by build.rs at variant/current.) + use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::net::Ipv4Addr; use crate::modeled_types::{ - Identifier, KubernetesClusterName, KubernetesLabelKey, KubernetesLabelValue, - KubernetesTaintValue, SingleLineString, Url, ValidBase64, + KubernetesClusterName, KubernetesLabelKey, KubernetesLabelValue, KubernetesTaintValue, + SingleLineString, Url, ValidBase64, }; -///// Primary user-visible settings - // Note: fields are marked with skip_serializing_if=Option::is_none so that settings GETs don't // show field=null for everything that isn't set in the relevant group of settings. -// Note: we have to use 'rename' here because the top-level Settings structure is the only one -// that uses its name in serialization; internal structures use the field name that poitns to it -#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename = "settings", rename_all = "kebab-case")] -pub struct Settings { - #[serde(skip_serializing_if = "Option::is_none")] - pub timezone: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub hostname: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub kubernetes: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub updates: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub host_containers: Option>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub ntp: Option, -} - // Kubernetes related settings. The dynamic settings are retrieved from // IMDS via Sundog's child "Pluto". #[rustfmt::skip] diff --git a/workspaces/models/src/modeled_types/kubernetes.rs b/workspaces/models/src/modeled_types/kubernetes.rs new file mode 100644 index 00000000000..8bda0d8586b --- /dev/null +++ b/workspaces/models/src/modeled_types/kubernetes.rs @@ -0,0 +1,339 @@ +use lazy_static::lazy_static; +use regex::Regex; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +// Just need serde's Error in scope to get its trait methods +use serde::de::Error as _; +use snafu::ensure; +use std::borrow::Borrow; +use std::convert::TryFrom; +use std::fmt; +use std::ops::Deref; +use super::error; + +/// KubernetesName represents a string that contains a valid Kubernetes resource name. It stores +/// the original string and makes it accessible through standard traits. +// https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesName { + inner: String, +} + +lazy_static! { + pub(crate) static ref KUBERNETES_NAME: Regex = Regex::new(r"^[0-9a-z.-]{1,253}$").unwrap(); +} + +impl TryFrom<&str> for KubernetesName { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + KUBERNETES_NAME.is_match(input), + error::Pattern { + thing: "Kubernetes name", + pattern: KUBERNETES_NAME.clone(), + input + } + ); + Ok(KubernetesName { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesName, "KubernetesName"); + +#[cfg(test)] +mod test_kubernetes_name { + use super::KubernetesName; + use std::convert::TryFrom; + + #[test] + fn good_names() { + for ok in &["howdy", "42", "18-eighteen."] { + KubernetesName::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_names() { + for err in &["", "HOWDY", "@", "hi/there", &"a".repeat(254)] { + KubernetesName::try_from(*err).unwrap_err(); + } + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesLabelKey represents a string that contains a valid Kubernetes label key. It stores +/// the original string and makes it accessible through standard traits. +// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesLabelKey { + inner: String, +} + +lazy_static! { + pub(crate) static ref KUBERNETES_LABEL_KEY: Regex = Regex::new( + r"(?x)^ + ( # optional prefix + [[:alnum:].-]{1,253}/ # DNS label characters followed by slash + )? + [[:alnum:]] # at least one alphanumeric + ( + ([[:alnum:]._-]{0,61})? # more characters allowed in middle + [[:alnum:]] # have to end with alphanumeric + )? + $" + ) + .unwrap(); +} + +impl TryFrom<&str> for KubernetesLabelKey { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + KUBERNETES_LABEL_KEY.is_match(input), + error::BigPattern { + thing: "Kubernetes label key", + input + } + ); + Ok(KubernetesLabelKey { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesLabelKey, "KubernetesLabelKey"); + +#[cfg(test)] +mod test_kubernetes_label_key { + use super::KubernetesLabelKey; + use std::convert::TryFrom; + + #[test] + fn good_keys() { + for ok in &[ + "no-prefix", + "have.a/prefix", + "more-chars_here.now", + &"a".repeat(63), + &format!("{}/{}", "a".repeat(253), "name"), + ] { + KubernetesLabelKey::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_keys() { + for err in &[ + ".bad", + "bad.", + &"a".repeat(64), + &format!("{}/{}", "a".repeat(254), "name"), + ] { + KubernetesLabelKey::try_from(*err).unwrap_err(); + } + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesLabelValue represents a string that contains a valid Kubernetes label value. It +/// stores the original string and makes it accessible through standard traits. +// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesLabelValue { + inner: String, +} + +lazy_static! { + pub(crate) static ref KUBERNETES_LABEL_VALUE: Regex = Regex::new( + r"(?x) + ^$ | # may be empty, or: + ^ + [[:alnum:]] # at least one alphanumeric + ( + ([[:alnum:]._-]{0,61})? # more characters allowed in middle + [[:alnum:]] # have to end with alphanumeric + )? + $ + " + ) + .unwrap(); +} + +impl TryFrom<&str> for KubernetesLabelValue { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + KUBERNETES_LABEL_VALUE.is_match(input), + error::BigPattern { + thing: "Kubernetes label value", + input + } + ); + Ok(KubernetesLabelValue { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesLabelValue, "KubernetesLabelValue"); + +#[cfg(test)] +mod test_kubernetes_label_value { + use super::KubernetesLabelValue; + use std::convert::TryFrom; + + #[test] + fn good_values() { + for ok in &["", "more-chars_here.now", &"a".repeat(63)] { + KubernetesLabelValue::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_values() { + for err in &[".bad", "bad.", &"a".repeat(64)] { + KubernetesLabelValue::try_from(*err).unwrap_err(); + } + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesTaintValue represents a string that contains a valid Kubernetes taint value, which is +/// like a label value, plus a colon, plus an "effect". It stores the original string and makes it +/// accessible through standard traits. +/// +/// Note: Kubelet won't launch if you specify an effect it doesn't know about, but we don't want to +/// gatekeep all possible values, so be careful. +// Note: couldn't find an exact spec for this. Cobbling things together, and guessing a bit as to +// the syntax of the effect. +// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set +// https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesTaintValue { + inner: String, +} + +lazy_static! { + pub(crate) static ref KUBERNETES_TAINT_VALUE: Regex = Regex::new( + r"(?x)^ + [[:alnum:]] # at least one alphanumeric + ( + ([[:alnum:]._-]{0,61})? # more characters allowed in middle + [[:alnum:]] # have to end with alphanumeric + )? + : # separate the label value from the effect + [[:alnum:]]{1,253} # effect + $" + ) + .unwrap(); +} + +impl TryFrom<&str> for KubernetesTaintValue { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + KUBERNETES_TAINT_VALUE.is_match(input), + error::BigPattern { + thing: "Kubernetes taint value", + input + } + ); + Ok(KubernetesTaintValue { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesTaintValue, "KubernetesTaintValue"); + +#[cfg(test)] +mod test_kubernetes_taint_value { + use super::KubernetesTaintValue; + use std::convert::TryFrom; + + #[test] + fn good_values() { + // All the examples from the docs linked above + for ok in &[ + "value:NoSchedule", + "value:PreferNoSchedule", + "value:NoExecute", + ] { + KubernetesTaintValue::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_values() { + for err in &[".bad", "bad.", &"a".repeat(254), "value:", ":effect"] { + KubernetesTaintValue::try_from(*err).unwrap_err(); + } + } +} + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesClusterName represents a string that contains a valid Kubernetes cluster name. It +/// stores the original string and makes it accessible through standard traits. +// Note: I was unable to find the rules for cluster naming. We know they have to fit into label +// values, because of the common cluster-name label, but they also can't be empty. This combines +// those two characteristics into a new type, until we find an explicit syntax. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesClusterName { + inner: String, +} + +impl TryFrom<&str> for KubernetesClusterName { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + !input.is_empty(), + error::InvalidClusterName { + name: input, + msg: "must not be empty" + } + ); + ensure!( + KubernetesLabelValue::try_from(input).is_ok(), + error::InvalidClusterName { + name: input, + msg: "cluster names must be valid Kubernetes label values" + } + ); + + Ok(KubernetesClusterName { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesClusterName, "KubernetesClusterName"); + +#[cfg(test)] +mod test_kubernetes_cluster_name { + use super::KubernetesClusterName; + use std::convert::TryFrom; + + #[test] + fn good_cluster_names() { + for ok in &["more-chars_here.now", &"a".repeat(63)] { + KubernetesClusterName::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_alues() { + for err in &["", ".bad", "bad.", &"a".repeat(64)] { + KubernetesClusterName::try_from(*err).unwrap_err(); + } + } +} diff --git a/workspaces/models/src/modeled_types/mod.rs b/workspaces/models/src/modeled_types/mod.rs new file mode 100644 index 00000000000..b40fa87d31d --- /dev/null +++ b/workspaces/models/src/modeled_types/mod.rs @@ -0,0 +1,127 @@ +//! This module contains data types that can be used in the model when special input/output +//! (ser/de) behavior is desired. For example, the ValidBase64 type can be used for a model field +//! when we don't even want to accept an API call with invalid base64 data. + +// The pattern in this module is to make a struct and implement TryFrom<&str> with code that does +// necessary checks and returns the struct. Other traits that treat the struct like a string can +// be implemented for you with the string_impls_for macro. + +pub mod error { + use regex::Regex; + use snafu::Snafu; + + #[derive(Debug, Snafu)] + #[snafu(visibility = "pub(super)")] + pub enum Error { + #[snafu(display("Can't create SingleLineString containing line terminator"))] + StringContainsLineTerminator, + + #[snafu(display("Invalid base64 input: {}", source))] + InvalidBase64 { source: base64::DecodeError }, + + #[snafu(display( + "Identifiers may only contain ASCII alphanumerics plus hyphens, received '{}'", + input + ))] + InvalidIdentifier { input: String }, + + #[snafu(display("Given invalid URL '{}'", input))] + InvalidUrl { input: String }, + + #[snafu(display("{} must match '{}', given: {}", thing, pattern, input))] + Pattern { + thing: String, + pattern: Regex, + input: String, + }, + + // Some regexes are too big to usefully display in an error. + #[snafu(display("{} given invalid input: {}", thing, input))] + BigPattern { thing: String, input: String }, + + #[snafu(display("Given invalid cluster name '{}': {}", name, msg))] + InvalidClusterName { name: String, msg: String }, + } +} + +/// Helper macro for implementing the common string-like traits for a modeled type. +/// Pass the name of the type, and the name of the type in quotes (to be used in string error +/// messages, etc.). +macro_rules! string_impls_for { + ($for:ident, $for_str:expr) => { + impl TryFrom for $for { + type Error = $crate::modeled_types::error::Error; + + fn try_from(input: String) -> Result { + Self::try_from(input.as_ref()) + } + } + + impl<'de> Deserialize<'de> for $for { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let original = String::deserialize(deserializer)?; + Self::try_from(original).map_err(|e| { + D::Error::custom(format!("Unable to deserialize into {}: {}", $for_str, e)) + }) + } + } + + /// We want to serialize the original string back out, not our structure, which is just there to + /// force validation. + impl Serialize for $for { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.inner) + } + } + + impl Deref for $for { + type Target = str; + fn deref(&self) -> &Self::Target { + &self.inner + } + } + + impl Borrow for $for { + fn borrow(&self) -> &String { + &self.inner + } + } + + impl Borrow for $for { + fn borrow(&self) -> &str { + &self.inner + } + } + + impl AsRef for $for { + fn as_ref(&self) -> &str { + &self.inner + } + } + + impl fmt::Display for $for { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.inner) + } + } + + impl From<$for> for String { + fn from(x: $for) -> Self { + x.inner + } + } + }; +} + +// Must be after macro definition +mod kubernetes; +mod shared; + +pub use kubernetes::*; +pub use shared::*; diff --git a/workspaces/models/aws-dev/modeled_types.rs b/workspaces/models/src/modeled_types/shared.rs similarity index 65% rename from workspaces/models/aws-dev/modeled_types.rs rename to workspaces/models/src/modeled_types/shared.rs index 1930cfe7d0f..7fd6229c010 100644 --- a/workspaces/models/aws-dev/modeled_types.rs +++ b/workspaces/models/src/modeled_types/shared.rs @@ -1,123 +1,54 @@ -//! This module contains data types that can be used in the model when special input/output -//! (ser/de) behavior is desired. For example, the SingleLineString type can be used for a model field -//! when we don't even want to accept an API call with multiple lines in the input. - -// The pattern in this file is to make a struct and implement TryFrom<&str> with code that does -// necessary checks and returns the struct. Other traits that treat the struct like a string can -// be implemented for you with the string_impls_for macro. - use serde::{Deserialize, Deserializer, Serialize, Serializer}; // Just need serde's Error in scope to get its trait methods use serde::de::Error as _; -use snafu::ensure; +use snafu::{ensure, ResultExt}; use std::borrow::Borrow; use std::convert::TryFrom; use std::fmt; use std::ops::Deref; +use super::error; -pub mod error { - use snafu::Snafu; - - #[derive(Debug, Snafu)] - #[snafu(visibility = "pub(super)")] - pub enum Error { - #[snafu(display("Can't create SingleLineString containing line terminator"))] - StringContainsLineTerminator, - - #[snafu(display("Invalid base64 input: {}", source))] - InvalidBase64 { source: base64::DecodeError }, - - #[snafu(display( - "Identifiers may only contain ASCII alphanumerics plus hyphens, received '{}'", - input - ))] - InvalidIdentifier { input: String }, - - #[snafu(display("Given invalid URL '{}'", input))] - InvalidUrl { input: String }, +/// ValidBase64 can only be created by deserializing from valid base64 text. It stores the +/// original text, not the decoded form. Its purpose is input validation, namely being used as a +/// field in a model structure so that you don't even accept a request with a field that has +/// invalid base64. +// Note: we use the default base64::STANDARD config which uses/allows "=" padding. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct ValidBase64 { + inner: String, +} - // Some regexes are too big to usefully display in an error. - #[snafu(display("{} given invalid input: {}", thing, input))] - BigPattern { thing: String, input: String }, +/// Validate base64 format before we accept the input. +impl TryFrom<&str> for ValidBase64 { + type Error = error::Error; - #[snafu(display("Given invalid cluster name '{}': {}", name, msg))] - InvalidClusterName { name: String, msg: String }, + fn try_from(input: &str) -> Result { + base64::decode(&input).context(error::InvalidBase64)?; + Ok(ValidBase64 { + inner: input.to_string(), + }) } } -/// Helper macro for implementing the common string-like traits for a modeled type. -/// Pass the name of the type, and the name of the type in quotes (to be used in string error -/// messages, etc.). -macro_rules! string_impls_for { - ($for:ident, $for_str:expr) => { - impl TryFrom for $for { - type Error = error::Error; - - fn try_from(input: String) -> Result { - Self::try_from(input.as_ref()) - } - } - - impl<'de> Deserialize<'de> for $for { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let original = String::deserialize(deserializer)?; - Self::try_from(original).map_err(|e| { - D::Error::custom(format!("Unable to deserialize into {}: {}", $for_str, e)) - }) - } - } - - /// We want to serialize the original string back out, not our structure, which is just there to - /// force validation. - impl Serialize for $for { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.inner) - } - } - - impl Deref for $for { - type Target = str; - fn deref(&self) -> &Self::Target { - &self.inner - } - } +string_impls_for!(ValidBase64, "ValidBase64"); - impl Borrow for $for { - fn borrow(&self) -> &String { - &self.inner - } - } - - impl Borrow for $for { - fn borrow(&self) -> &str { - &self.inner - } - } - - impl AsRef for $for { - fn as_ref(&self) -> &str { - &self.inner - } - } +#[cfg(test)] +mod test_valid_base64 { + use super::ValidBase64; + use std::convert::TryFrom; - impl fmt::Display for $for { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.inner) - } - } + #[test] + fn valid_base64() { + let v = ValidBase64::try_from("aGk=").unwrap(); + let decoded_bytes = base64::decode(v.as_ref()).unwrap(); + let decoded = std::str::from_utf8(&decoded_bytes).unwrap(); + assert_eq!(decoded, "hi"); + } - impl From<$for> for String { - fn from(x: $for) -> Self { - x.inner - } - } - }; + #[test] + fn invalid_base64() { + assert!(ValidBase64::try_from("invalid base64").is_err()); + } } // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= diff --git a/workspaces/models/current/.keep b/workspaces/models/src/variant/.keep similarity index 100% rename from workspaces/models/current/.keep rename to workspaces/models/src/variant/.keep diff --git a/workspaces/models/src/variant_mod.rs b/workspaces/models/src/variant_mod.rs new file mode 100644 index 00000000000..1ea50dd8372 --- /dev/null +++ b/workspaces/models/src/variant_mod.rs @@ -0,0 +1,4 @@ +// This is linked into place at variant/mod.rs because the build system mounts a temporary +// directory at variant/ - see README.md. +mod current; +pub use current::*;