From c422738d48c2da8efe56eda910b82bbcb24d61a0 Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Mon, 16 Sep 2019 14:17:08 -0700 Subject: [PATCH 01/10] update_metadata: Split out metadata definition from Updog. Signed-off-by: Samuel Mendoza-Jonas --- workspaces/Cargo.lock | 124 ++++++++++++++++++ workspaces/Cargo.toml | 1 + workspaces/deny.toml | 1 + workspaces/updater/update_metadata/Cargo.toml | 21 +++ .../{updog => update_metadata}/src/de.rs | 4 +- .../updater/update_metadata/src/error.rs | 84 ++++++++++++ workspaces/updater/update_metadata/src/lib.rs | 86 ++++++++++++ .../{updog => update_metadata}/src/se.rs | 9 +- workspaces/updater/updog/Cargo.toml | 9 +- workspaces/updater/updog/src/error.rs | 8 +- workspaces/updater/updog/src/main.rs | 82 +----------- 11 files changed, 342 insertions(+), 87 deletions(-) create mode 100644 workspaces/updater/update_metadata/Cargo.toml rename workspaces/updater/{updog => update_metadata}/src/de.rs (97%) create mode 100644 workspaces/updater/update_metadata/src/error.rs create mode 100644 workspaces/updater/update_metadata/src/lib.rs rename workspaces/updater/{updog => update_metadata}/src/se.rs (70%) diff --git a/workspaces/Cargo.lock b/workspaces/Cargo.lock index fe4a300eab6..36ce4a43a49 100644 --- a/workspaces/Cargo.lock +++ b/workspaces/Cargo.lock @@ -1280,6 +1280,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "libc" @@ -1357,6 +1360,14 @@ name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "matches" version = "0.1.8" @@ -1669,6 +1680,14 @@ dependencies = [ "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "owning_ref" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parking_lot" version = "0.9.0" @@ -2052,6 +2071,16 @@ dependencies = [ "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "regex-automata" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex-syntax" version = "0.6.12" @@ -2516,6 +2545,11 @@ name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "stable_deref_trait" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "storewolf" version = "0.1.0" @@ -2996,6 +3030,61 @@ dependencies = [ "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tracing" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tracing-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tracing-subscriber" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "matchers 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-log 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "trust-dns-proto" version = "0.7.4" @@ -3139,6 +3228,21 @@ name = "untrusted" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "update_metadata" +version = "0.1.0" +dependencies = [ + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "data_store_version 0.1.0", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.101 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_plain 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "snafu 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "updog" version = "0.1.0" @@ -3156,9 +3260,14 @@ dependencies = [ "signpost 0.1.0", "simplelog 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "snafu 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "tough 0.1.0", + "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-subscriber 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "update_metadata 0.1.0", ] [[package]] @@ -3186,6 +3295,11 @@ name = "utf8-cstr" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "utf8-ranges" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "uuid" version = "0.7.4" @@ -3548,6 +3662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lz4 1.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "43c94a9f09a60017f373020cc93d4291db4cd92b0db64ff25927f27d09dc23d5" "checksum lz4-sys 1.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "20ab022822e9331c58d373acdd6b98085bace058ac6837b8266f213a2fccdafe" "checksum maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +"checksum matchers 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum md5 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e6bcd6433cff03a4bfc3d9834d504467db1f1cf6d0ea765d37d330249ed629d" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" @@ -3575,6 +3690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" "checksum openssl-src 111.6.0+1.1.1d (registry+https://github.com/rust-lang/crates.io-index)" = "b9c2da1de8a7a3f860919c01540b03a6db16de042405a8a07a5e9d0b4b825d9c" "checksum openssl-sys 0.9.51 (registry+https://github.com/rust-lang/crates.io-index)" = "ba24190c8f0805d3bd2ce028f439fe5af1d55882bbe6261bed1dbc93b50dd6b1" +"checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" "checksum pem 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39eb474073dfddbf7156515344266245d91ce698ddbf15e0498cef22b836f45a" @@ -3616,6 +3732,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecedbca3bf205f8d8f5c2b44d83cd0690e39ee84b951ed649e9f1841132b66d" "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" +"checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650" @@ -3658,6 +3775,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85" "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +"checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum structopt 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4f66a4c0ddf7aee4677995697366de0749b0139057342eccbb609b12d0affc" @@ -3694,6 +3812,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" "checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" "checksum toml 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c7aabe75941d914b72bf3e5d3932ed92ce0664d49d8432305a8b547c37227724" +"checksum tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ff4e4f59e752cb3beb5b61c6d5e11191c7946231ba84faec2902c9efdd8691c5" +"checksum tracing-attributes 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a4263b12c3d3c403274493eb805966093b53214124796552d674ca1dd5d27c2b" +"checksum tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bc913647c520c959b6d21e35ed8fa6984971deca9f0a2fcb8c51207e0c56af1d" +"checksum tracing-log 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +"checksum tracing-subscriber 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "192ca16595cdd0661ce319e8eede9c975f227cdaabc4faaefdc256f43d852e45" "checksum trust-dns-proto 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5559ebdf6c2368ddd11e20b11d6bbaf9e46deb803acd7815e93f5a7b4a6d2901" "checksum trust-dns-resolver 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c9992e58dba365798803c0b91018ff6c8d3fc77e06977c4539af2a6bfe0a039" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" @@ -3711,6 +3834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" "checksum url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "75b414f6c464c879d7f9babf951f23bc3743fb7313c081b2e6ca719067ea9d61" "checksum utf8-cstr 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628" +"checksum utf8-ranges 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba" "checksum uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a" "checksum vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "33dd455d0f96e90a75803cfeb7f948768c08d70a6de9a8d2362461935698bf95" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" diff --git a/workspaces/Cargo.toml b/workspaces/Cargo.toml index 3e476a51f68..f86fcd26c3e 100644 --- a/workspaces/Cargo.toml +++ b/workspaces/Cargo.toml @@ -27,6 +27,7 @@ members = [ "updater/signpost", "updater/tough", "updater/tough_schema", + "updater/update_metadata", "updater/updog", "tuftool", diff --git a/workspaces/deny.toml b/workspaces/deny.toml index 95e75a7fbe9..ff381f1f6b1 100644 --- a/workspaces/deny.toml +++ b/workspaces/deny.toml @@ -52,6 +52,7 @@ skip = [ { name = "tough", licenses = [] }, { name = "tough_schema", licenses = [] }, { name = "tuftool", licenses = [] }, + { name = "update_metadata", licenses = [] }, { name = "updog", licenses = [] }, { name = "webpki-roots", licenses = [] }, diff --git a/workspaces/updater/update_metadata/Cargo.toml b/workspaces/updater/update_metadata/Cargo.toml new file mode 100644 index 00000000000..92bafc72937 --- /dev/null +++ b/workspaces/updater/update_metadata/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "update_metadata" +version = "0.1.0" +authors = ["Samuel Mendoza-Jonas "] +edition = "2018" +publish = false + +[dependencies] +chrono = { version = "0.4.9", features = ["serde"] } +data_store_version = { path = "../../api/data_store_version" } +rand = "0.7.0" +regex = "1.1" +semver = { version = "0.9.0", features = ["serde"] } +serde = { version = "1.0.100", features = ["derive"] } +serde_json = "1.0.40" +serde_plain = "0.3.0" +snafu = "0.5.0" + +[lib] +name = "update_metadata" +path = "src/lib.rs" diff --git a/workspaces/updater/updog/src/de.rs b/workspaces/updater/update_metadata/src/de.rs similarity index 97% rename from workspaces/updater/updog/src/de.rs rename to workspaces/updater/update_metadata/src/de.rs index b18dc180647..2d3e71c0579 100644 --- a/workspaces/updater/updog/src/de.rs +++ b/workspaces/updater/update_metadata/src/de.rs @@ -64,7 +64,7 @@ where D: Deserializer<'de>, { fn parse_versions(key: &str) -> Result<(&str, &str), error::Error> { - let r = Regex::new(r"\((?P[0-9.]+),[ ]+(?P[0-9.]+)\)"); + let r = Regex::new(r"\((?P[0-9.]+),[ ]*(?P[0-9.]+)\)"); if let Ok(regex) = r { if let Some(captures) = regex.captures(&key) { @@ -126,7 +126,7 @@ where /// Converts the key and value into a Version/DVersion pair before insertion and /// catches duplicates -pub(crate) fn deserialize_datastore_version<'de, D>( +pub(crate) fn deserialize_datastore_map<'de, D>( deserializer: D, ) -> Result, D::Error> where diff --git a/workspaces/updater/update_metadata/src/error.rs b/workspaces/updater/update_metadata/src/error.rs new file mode 100644 index 00000000000..45fb0fd1013 --- /dev/null +++ b/workspaces/updater/update_metadata/src/error.rs @@ -0,0 +1,84 @@ +#![allow(clippy::default_trait_access)] + +use data_store_version::Version as DataVersion; +use snafu::{Backtrace, Snafu}; + +pub type Result = std::result::Result; + +#[derive(Debug, Snafu)] +#[snafu(visibility = "pub(crate)")] +pub enum Error { + #[snafu(display("Bad bound field: {}", bound_str))] + BadBound { + backtrace: Backtrace, + source: std::num::ParseIntError, + bound_str: String, + }, + + #[snafu(display("Invalid bound start: {}", key))] + BadBoundKey { + source: std::num::ParseIntError, + key: String, + backtrace: Backtrace, + }, + + #[snafu(display("Could not parse datastore version: {}", key))] + BadDataVersion { backtrace: Backtrace, key: String }, + + #[snafu(display("Could not parse image version: {} - {}", key, value))] + BadMapVersion { + backtrace: Backtrace, + key: String, + value: String, + }, + + #[snafu(display("Duplicate key ID: {}", keyid))] + DuplicateKeyId { backtrace: Backtrace, keyid: u64 }, + + #[snafu(display("Duplicate version key: {}", key))] + DuplicateVersionKey { backtrace: Backtrace, key: String }, + + #[snafu(display("Failed to parse updates manifest: {}", source))] + ManifestParse { + source: serde_json::Error, + backtrace: Backtrace, + }, + + #[snafu(display("Missing datastore version in metadata: {:?}", version))] + MissingDataVersion { + backtrace: Backtrace, + version: DataVersion, + }, + + #[snafu(display("Image version missing datastore mapping: {}", version))] + MissingMapping { + backtrace: Backtrace, + version: String, + }, + + #[snafu(display( + "Reached end of migration chain at {} but target is {}", + current, + target + ))] + MissingMigration { + backtrace: Backtrace, + current: DataVersion, + target: DataVersion, + }, + + #[snafu(display("Missing version in metadata: {}", version))] + MissingVersion { + backtrace: Backtrace, + version: String, + }, + + #[snafu(display("This host is not part of any wave"))] + NoWave { backtrace: Backtrace }, + + #[snafu(display("Failed to serialize update information: {}", source))] + UpdateSerialize { + source: serde_json::Error, + backtrace: Backtrace, + }, +} diff --git a/workspaces/updater/update_metadata/src/lib.rs b/workspaces/updater/update_metadata/src/lib.rs new file mode 100644 index 00000000000..a1a36fc5b84 --- /dev/null +++ b/workspaces/updater/update_metadata/src/lib.rs @@ -0,0 +1,86 @@ +#![warn(clippy::pedantic)] + +mod de; +pub mod error; +mod se; + +use chrono::{DateTime, Duration, Utc}; +use data_store_version::Version as DVersion; +use rand::{thread_rng, Rng}; +use semver::Version; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::ops::Bound::{Excluded, Included}; + +pub const MAX_SEED: u32 = 2048; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Images { + pub boot: String, + pub root: String, + pub hash: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Update { + pub flavor: String, + pub arch: String, + pub version: Version, + pub max_version: Version, + #[serde(deserialize_with = "de::deserialize_bound")] + pub waves: BTreeMap>, + pub images: Images, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Manifest { + pub updates: Vec, + #[serde(deserialize_with = "de::deserialize_migration")] + #[serde(serialize_with = "se::serialize_migration")] + pub migrations: BTreeMap<(DVersion, DVersion), Vec>, + #[serde(deserialize_with = "de::deserialize_datastore_map")] + #[serde(serialize_with = "se::serialize_datastore_map")] + pub datastore_versions: BTreeMap, +} + +impl Update { + pub fn update_wave(&self, seed: u32) -> Option<&DateTime> { + if let Some((_, wave)) = self.waves.range((Included(0), Included(seed))).last() { + return Some(wave); + } + None + } + + pub fn update_ready(&self, seed: u32) -> bool { + // Has this client's wave started + if let Some(wave) = self.update_wave(seed) { + return *wave <= Utc::now(); + } + + // Alternately have all waves passed + if let Some((_, wave)) = self.waves.iter().last() { + return *wave <= Utc::now(); + } + + // Or there are no waves + true + } + + pub fn jitter(&self, seed: u32) -> Option> { + let prev = self.update_wave(seed); + let next = self + .waves + .range((Excluded(seed), Excluded(MAX_SEED))) + .next(); + if let (Some(start), Some(end)) = (prev, next) { + if Utc::now() < *end.1 { + let mut rng = thread_rng(); + #[allow(clippy::cast_sign_loss)] + let range = (end.1.timestamp() - start.timestamp()) as u64; + let jitter = Duration::seconds(rng.gen_range(1, range) as i64); + return Some(*start + jitter); + } + } + None + } +} diff --git a/workspaces/updater/updog/src/se.rs b/workspaces/updater/update_metadata/src/se.rs similarity index 70% rename from workspaces/updater/updog/src/se.rs rename to workspaces/updater/update_metadata/src/se.rs index cd19dbfe76d..7602b485aa2 100644 --- a/workspaces/updater/updog/src/se.rs +++ b/workspaces/updater/update_metadata/src/se.rs @@ -12,7 +12,11 @@ where { let mut map = BTreeMap::new(); for ((from, to), val) in value { - let key = format!("({},{})", from, to); + let key = format!( + "({},{})", + from.to_string().trim_start_matches('v'), + to.to_string().trim_start_matches('v') + ); map.insert(key, val); } map.serialize(serializer) @@ -27,7 +31,8 @@ where { let mut map = BTreeMap::new(); for (image, datastore) in value { - map.insert(image.to_string(), datastore.to_string()); + let datastore = String::from(datastore.to_string().trim_start_matches('v')); + map.insert(image.to_string(), datastore); } map.serialize(serializer) } diff --git a/workspaces/updater/updog/Cargo.toml b/workspaces/updater/updog/Cargo.toml index 9340071d757..0132a45b0b7 100644 --- a/workspaces/updater/updog/Cargo.toml +++ b/workspaces/updater/updog/Cargo.toml @@ -13,7 +13,7 @@ lz4 = "1.23.1" rand = "0.7.0" regex = "1.1" semver = "0.9.0" -serde = "1.0.100" +serde = { version = "1.0.100", features = ["derive"] } serde_json = "1.0.40" serde_plain = "0.3.0" signpost = { path = "../signpost" } @@ -22,3 +22,10 @@ snafu = "0.5.0" time = "0.1" toml = "0.5.1" tough = { path = "../tough" } +tracing = "0.1" +tracing-subscriber = "0.1" +update_metadata = { path = "../update_metadata" } +structopt = "0.3" + +[dev-dependencies] +tempfile = "3.1.0" diff --git a/workspaces/updater/updog/src/error.rs b/workspaces/updater/updog/src/error.rs index 6068e85fc76..1b70c8f6ef8 100644 --- a/workspaces/updater/updog/src/error.rs +++ b/workspaces/updater/updog/src/error.rs @@ -244,11 +244,11 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Update wave has been missed"))] - WaveMissed { backtrace: Backtrace }, + #[snafu(display("Update wave start time missing"))] + WaveStartArg { backtrace: Backtrace }, - #[snafu(display("Update wave is pending"))] - WavePending { backtrace: Backtrace }, + #[snafu(display("Waves are not ordered: bound {} occurs before bound {}", next, wave))] + WavesUnordered { wave: u32, next: u32 }, #[snafu(display("Failed writing update data to disk: {}", source))] WriteUpdate { diff --git a/workspaces/updater/updog/src/main.rs b/workspaces/updater/updog/src/main.rs index 178f2c12165..35745fe5454 100644 --- a/workspaces/updater/updog/src/main.rs +++ b/workspaces/updater/updog/src/main.rs @@ -1,27 +1,23 @@ #![deny(rust_2018_idioms)] #![warn(clippy::pedantic)] -mod de; mod error; -mod se; use crate::error::Result; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use data_store_version::Version as DVersion; -use rand::{thread_rng, Rng}; use semver::Version; use serde::{Deserialize, Serialize}; use signpost::State; use simplelog::{Config as LogConfig, LevelFilter, TermLogger, TerminalMode}; use snafu::{ErrorCompat, OptionExt, ResultExt}; -use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; use std::io::{self, BufRead, BufReader}; -use std::ops::Bound::{Excluded, Included}; use std::path::Path; use std::process; use std::str::FromStr; use tough::{Limits, Repository, Settings}; +use update_metadata::{Manifest, Update}; #[cfg(target_arch = "x86_64")] const TARGET_ARCH: &str = "x86_64"; @@ -30,7 +26,6 @@ const TARGET_ARCH: &str = "aarch64"; const TRUSTED_ROOT_PATH: &str = "/usr/share/updog/root.json"; const MIGRATION_PATH: &str = "/var/lib/thar/datastore/migrations"; -const MAX_SEED: u32 = 2048; #[derive(Debug, Deserialize, PartialEq)] #[serde(rename_all = "kebab-case")] @@ -53,77 +48,6 @@ struct Config { // mode: Option<{Automatic, Managed, Disabled}> } -#[derive(Debug, Serialize, Deserialize)] -struct Images { - boot: String, - root: String, - hash: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct Update { - flavor: String, - arch: String, - version: Version, - max_version: Version, - #[serde(deserialize_with = "de::deserialize_bound")] - waves: BTreeMap>, - images: Images, -} - -impl Update { - pub fn update_wave(&self, seed: u32) -> Option<&DateTime> { - if let Some((_, wave)) = self.waves.range((Included(0), Included(seed))).last() { - return Some(wave); - } - None - } - - fn update_ready(&self, seed: u32) -> bool { - // Has this client's wave started - if let Some(wave) = self.update_wave(seed) { - return *wave <= Utc::now(); - } - - // Alternately have all waves passed - if let Some((_, wave)) = self.waves.iter().last() { - return *wave <= Utc::now(); - } - - // Or there are no waves - true - } - - pub fn jitter(&self, seed: u32) -> Option> { - let prev = self.update_wave(seed); - let next = self - .waves - .range((Excluded(seed), Excluded(MAX_SEED))) - .next(); - if let (Some(start), Some(end)) = (prev, next) { - if Utc::now() < *end.1 { - let mut rng = thread_rng(); - #[allow(clippy::cast_sign_loss)] - let range = (end.1.timestamp() - start.timestamp()) as u64; - let jitter = Duration::seconds(rng.gen_range(1, range) as i64); - return Some(*start + jitter); - } - } - None - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct Manifest { - updates: Vec, - #[serde(deserialize_with = "de::deserialize_migration")] - #[serde(serialize_with = "se::serialize_migration")] - migrations: BTreeMap<(DVersion, DVersion), Vec>, - #[serde(deserialize_with = "de::deserialize_datastore_version")] - #[serde(serialize_with = "se::serialize_datastore_map")] - datastore_versions: BTreeMap, -} - /// Prints a more specific message before exiting through usage(). fn usage_msg>(msg: S) -> ! { eprintln!("{}\n", msg.as_ref()); @@ -634,7 +558,9 @@ fn main() -> ! { mod tests { use super::*; use chrono::Duration as TestDuration; + use std::collections::BTreeMap; use std::str::FromStr; + use update_metadata::Images; #[test] fn test_manifest_json() { From 1cbb61e93a92c99f6b329720bc5c9dbd170cd75e Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Fri, 11 Oct 2019 14:27:13 -0700 Subject: [PATCH 02/10] update_metadata: Simplify update_wave() From: iliana destroyer of worlds Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/update_metadata/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workspaces/updater/update_metadata/src/lib.rs b/workspaces/updater/update_metadata/src/lib.rs index a1a36fc5b84..4f1ce67433c 100644 --- a/workspaces/updater/update_metadata/src/lib.rs +++ b/workspaces/updater/update_metadata/src/lib.rs @@ -45,10 +45,10 @@ pub struct Manifest { impl Update { pub fn update_wave(&self, seed: u32) -> Option<&DateTime> { - if let Some((_, wave)) = self.waves.range((Included(0), Included(seed))).last() { - return Some(wave); - } - None + self.waves + .range((Included(0), Included(seed))) + .last() + .map(|(_, wave)| wave) } pub fn update_ready(&self, seed: u32) -> bool { From 2d5ae6625d48d0dea22f7ea541d664dce3ed6061 Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Mon, 23 Sep 2019 16:06:18 -0700 Subject: [PATCH 03/10] updata: Add tool to generate update metadata Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/updog/src/bin/updata.rs | 343 +++++++++++++++++++++ workspaces/updater/updog/src/error.rs | 10 + 2 files changed, 353 insertions(+) create mode 100644 workspaces/updater/updog/src/bin/updata.rs diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs new file mode 100644 index 00000000000..51e596fd9d2 --- /dev/null +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -0,0 +1,343 @@ +#![warn(clippy::pedantic)] + +#[path = "../error.rs"] +mod error; + +use crate::error::Result; +use chrono::{DateTime, Utc}; +use data_store_version::Version as DVersion; +use semver::Version; +use snafu::{ErrorCompat, ResultExt}; +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; +use update_metadata::{Images, Manifest, Update}; + +#[derive(Debug, StructOpt)] +struct GeneralArgs { + // metadata file to create/modify + file: PathBuf, +} + +#[derive(Debug, StructOpt)] +struct AddUpdateArgs { + // metadata file to create/modify + file: PathBuf, + + // image 'flavor', eg. 'aws-k8s' + #[structopt(short = "f", long = "flavor")] + flavor: String, + + // image version + #[structopt(short = "v", long = "version")] + version: Version, + + // architecture image is built for + #[structopt(short = "a", long = "arch")] + arch: String, + + // corresponding datastore version for this image + #[structopt(short = "d", long = "data-version")] + datastore: DVersion, + + // maximum valid version + #[structopt(short = "m", long = "max-version")] + max_version: Version, + + // root image target name + #[structopt(short = "r", long = "root")] + root: String, + + // boot image target name + #[structopt(short = "b", long = "boot")] + boot: String, + + // verity "hash" image target name + #[structopt(short = "h", long = "hash")] + hash: String, +} + +impl AddUpdateArgs { + fn run(self) -> Result<()> { + let mut m: Manifest = match load_file(&self.file) { + Ok(m) => m, + _ => Manifest::default(), // TODO only if EEXIST + }; + m.datastore_versions + .insert(self.version.clone(), self.datastore); + let u = Update { + flavor: self.flavor, + arch: self.arch, + version: self.version, + max_version: self.max_version, + images: Images { + root: self.root, + boot: self.boot, + hash: self.hash, + }, + waves: BTreeMap::new(), + }; + update_max_version(&mut m, &u); + m.updates.push(u); + write_file(&self.file, &m) + } +} + +#[derive(Debug, StructOpt)] +struct RemoveUpdateArgs { + // metadata file to create/modify + file: PathBuf, + + // image 'flavor', eg. 'aws-k8s' + #[structopt(short = "l", long = "flavor")] + flavor: String, + + // image version + #[structopt(short = "v", long = "version")] + version: Version, + + // architecture image is built for + #[structopt(short = "a", long = "arch")] + arch: String, +} + +impl RemoveUpdateArgs { + fn run(&self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + m.updates.retain(|u| { + !(u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + }); + // Note: We don't revert the maximum version on removal + write_file(&self.file, &m) + } +} + +#[derive(Debug, StructOpt)] +struct WaveArgs { + // metadata file to create/modify + file: PathBuf, + + // image 'flavor', eg. 'aws-k8s' + #[structopt(short = "l", long = "flavor")] + flavor: String, + + // image version + #[structopt(short = "v", long = "version")] + version: Version, + + // architecture image is built for + #[structopt(short = "a", long = "arch")] + arch: String, + + // start bound id for this wave (0 <= x < 2048) + #[structopt(short = "b", long = "bound-id")] + bound: u32, + + // start time for this wave + #[structopt(short = "s", long = "start-time")] + start: Option>, +} + +impl WaveArgs { + fn add(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + let matching: Vec<&mut Update> = m + .updates + .iter_mut() + .filter(|u| u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + .collect(); + if matching.len() > 1 { + println!("Multiple matching updates for wave - this is weird but not a disaster"); + } + if let Some(start) = self.start { + for u in matching { + u.waves.insert(self.bound, start); + } + write_file(&self.file, &m) + } else { + error::WaveStartArg.fail() + } + } + + fn remove(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + let matching: Vec<&mut Update> = m + .updates + .iter_mut() + .filter(|u| u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + .collect(); + for u in matching { + u.waves.remove(&self.bound); + } + write_file(&self.file, &m) + } +} + +#[derive(Debug, StructOpt)] +struct MigrationArgs { + // metadata file to create/modify + file: PathBuf, + + // starting datastore version + #[structopt(short = "f", long = "from")] + from: DVersion, + + // target datastore version + #[structopt(short = "t", long = "to")] + to: DVersion, + + // whether to append to or replace any existing migration list + #[structopt(short, long)] + append: bool, + + // migration names + migrations: Vec, +} + +impl MigrationArgs { + fn add(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + let mut migrations = self.migrations; + if self.append { + if let Some(e) = m.migrations.remove(&(self.from, self.to)) { + migrations.extend_from_slice(&e); + } + } + m.migrations.insert((self.from, self.to), migrations); + write_file(&self.file, &m) + } + + fn remove(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + m.migrations.remove(&(self.from, self.to)); + write_file(&self.file, &m) + } +} + +#[derive(Debug, StructOpt)] +#[structopt(rename_all = "kebab-case")] +enum Command { + Init(GeneralArgs), + AddUpdate(AddUpdateArgs), + AddWave(WaveArgs), + AddMigration(MigrationArgs), + RemoveUpdate(RemoveUpdateArgs), + RemoveMigration(MigrationArgs), + RemoveWave(WaveArgs), + Validate(GeneralArgs), +} + +fn load_file(path: &Path) -> Result { + serde_json::from_reader(File::open(path).context(error::ManifestRead { path })?) + .context(error::ManifestParse) +} + +fn write_file(path: &Path, manifest: &Manifest) -> Result<()> { + let manifest = serde_json::to_string_pretty(&manifest).context(error::UpdateSerialize)?; + fs::write(path, &manifest).context(error::ConfigWrite { path })?; + Ok(()) +} + +/// Update the maximum version for all updates that match the architecture +/// and flavor of some new update. +fn update_max_version(m: &mut Manifest, new: &Update) { + let matching: Vec<&mut Update> = m + .updates + .iter_mut() + .filter(|u| u.arch == new.arch && u.flavor == new.flavor) + .collect(); + for u in matching { + u.max_version = new.max_version.clone(); + } +} + +fn main_inner() -> Result<()> { + match Command::from_args() { + Command::Init(args) => write_file(&args.file, &Manifest::default()), + Command::AddUpdate(args) => args.run(), + Command::AddWave(args) => args.add(), + Command::AddMigration(args) => args.add(), + Command::RemoveUpdate(args) => args.run(), + Command::RemoveWave(args) => args.remove(), + Command::RemoveMigration(args) => args.remove(), + Command::Validate(args) => match load_file(&args.file) { + Ok(_) => Ok(()), + Err(e) => Err(e), + }, + } +} + +fn main() -> ! { + std::process::exit(match main_inner() { + Ok(()) => 0, + Err(err) => { + eprintln!("{}", err); + if let Some(var) = std::env::var_os("RUST_BACKTRACE") { + if var != "0" { + if let Some(backtrace) = err.backtrace() { + eprintln!("\n{:?}", backtrace); + } + } + } + 1 + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + use tempfile::NamedTempFile; + + #[test] + fn max_versions() -> Result<()> { + let tmpfd = NamedTempFile::new().context(error::TmpFileCreate)?; + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.3").unwrap(), + max_version: Version::parse("1.2.3").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.5").unwrap(), + max_version: Version::parse("1.2.3").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.4").unwrap(), + max_version: Version::parse("1.2.4").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + + let m: Manifest = load_file(tmpfd.path())?; + for u in m.updates { + assert!(u.max_version == Version::parse("1.2.4").unwrap()); + } + Ok(()) + } +} diff --git a/workspaces/updater/updog/src/error.rs b/workspaces/updater/updog/src/error.rs index 1b70c8f6ef8..2f8cbf341ee 100644 --- a/workspaces/updater/updog/src/error.rs +++ b/workspaces/updater/updog/src/error.rs @@ -117,6 +117,13 @@ pub(crate) enum Error { backtrace: Backtrace, }, + #[snafu(display("Failed to read manifest file {}: {}", path.display(), source))] + ManifestRead { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + #[snafu(display("Metadata error: {}", source))] Metadata { source: tough::error::Error, @@ -250,6 +257,9 @@ pub(crate) enum Error { #[snafu(display("Waves are not ordered: bound {} occurs before bound {}", next, wave))] WavesUnordered { wave: u32, next: u32 }, + #[snafu(display("Update wave start time missing"))] + WaveStartArg { backtrace: Backtrace }, + #[snafu(display("Failed writing update data to disk: {}", source))] WriteUpdate { source: std::io::Error, From e2e4b84d60a13220472f4e9fec801b484b6b88fb Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Tue, 24 Sep 2019 10:34:09 -0700 Subject: [PATCH 04/10] updata: Add set-max-version command Add the set-max-version command to update the maximum version field across all updates in the manifest, optionally filtering by arch and flavor. Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/updog/src/bin/updata.rs | 42 ++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs index 51e596fd9d2..e56c76b086a 100644 --- a/workspaces/updater/updog/src/bin/updata.rs +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -78,7 +78,7 @@ impl AddUpdateArgs { }, waves: BTreeMap::new(), }; - update_max_version(&mut m, &u); + update_max_version(&mut m, &u.max_version, Some(&u.arch), Some(&u.flavor)); m.updates.push(u); write_file(&self.file, &m) } @@ -215,6 +215,24 @@ impl MigrationArgs { } } +#[derive(Debug, StructOpt)] +struct MaxVersionArgs { + // metadata file to create/modify + file: PathBuf, + + // maximum valid version + #[structopt(short, long)] + max_version: Version, +} + +impl MaxVersionArgs { + fn run(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + update_max_version(&mut m, &self.max_version, None, None); + write_file(&self.file, &m) + } +} + #[derive(Debug, StructOpt)] #[structopt(rename_all = "kebab-case")] enum Command { @@ -222,6 +240,7 @@ enum Command { AddUpdate(AddUpdateArgs), AddWave(WaveArgs), AddMigration(MigrationArgs), + SetMaxVersion(MaxVersionArgs), RemoveUpdate(RemoveUpdateArgs), RemoveMigration(MigrationArgs), RemoveWave(WaveArgs), @@ -239,16 +258,26 @@ fn write_file(path: &Path, manifest: &Manifest) -> Result<()> { Ok(()) } -/// Update the maximum version for all updates that match the architecture -/// and flavor of some new update. -fn update_max_version(m: &mut Manifest, new: &Update) { +/// Update the maximum version for all updates that optionally match the +/// architecture and flavor of some new update. +fn update_max_version( + m: &mut Manifest, + version: &Version, + arch: Option<&str>, + flavor: Option<&str>, +) { let matching: Vec<&mut Update> = m .updates .iter_mut() - .filter(|u| u.arch == new.arch && u.flavor == new.flavor) + .filter(|u| match (arch, flavor) { + (Some(arch), Some(flavor)) => u.arch == arch && u.flavor == flavor, + (Some(arch), None) => u.arch == arch, + (None, Some(flavor)) => u.flavor == flavor, + _ => true, + }) .collect(); for u in matching { - u.max_version = new.max_version.clone(); + u.max_version = version.clone(); } } @@ -258,6 +287,7 @@ fn main_inner() -> Result<()> { Command::AddUpdate(args) => args.run(), Command::AddWave(args) => args.add(), Command::AddMigration(args) => args.add(), + Command::SetMaxVersion(args) => args.run(), Command::RemoveUpdate(args) => args.run(), Command::RemoveWave(args) => args.remove(), Command::RemoveMigration(args) => args.remove(), From 407c174f509b6d1072dcea4a91f3aef161109d41 Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Tue, 24 Sep 2019 10:36:59 -0700 Subject: [PATCH 05/10] updata: Add --cleanup flag to remove-update. If a removed update was the final reference to a datastore version and --cleanup is set, remove that version:datastore mapping from the manifest. Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/updog/src/bin/updata.rs | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs index e56c76b086a..5f6b291c730 100644 --- a/workspaces/updater/updog/src/bin/updata.rs +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -100,6 +100,14 @@ struct RemoveUpdateArgs { // architecture image is built for #[structopt(short = "a", long = "arch")] arch: String, + + // Whether to clean up datastore mappings that no longer reference an + // existing update. Migration paths for such datastore versions are + // preserved. + // This should _only_ be used if there are no existing users of the + // specified Thar image version. + #[structopt(short, long)] + cleanup: bool, } impl RemoveUpdateArgs { @@ -108,6 +116,22 @@ impl RemoveUpdateArgs { m.updates.retain(|u| { !(u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) }); + if self.cleanup { + let remaining: Vec<&Update> = m + .updates + .iter() + .filter(|u| u.version == self.version) + .collect(); + if remaining.is_empty() { + m.datastore_versions.remove(&self.version); + } else { + println!( + "Cleanup skipped; {} {} updates remain", + remaining.len(), + self.version + ); + } + } // Note: We don't revert the maximum version on removal write_file(&self.file, &m) } @@ -370,4 +394,65 @@ mod tests { } Ok(()) } + + #[test] + fn datastore_mapping() -> Result<()> { + let tmpfd = NamedTempFile::new().context(error::TmpFileCreate)?; + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.3").unwrap(), + max_version: Version::parse("1.2.3").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.5").unwrap(), + max_version: Version::parse("1.2.3").unwrap(), + datastore: DVersion::from_str("1.1").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.4").unwrap(), + max_version: Version::parse("1.2.4").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + + // TODO this needs to test against ARCH and FLAVOR not being considered + RemoveUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.4").unwrap(), + cleanup: true, + } + .run() + .unwrap(); + + let m: Manifest = load_file(tmpfd.path())?; + assert!(m + .datastore_versions + .contains_key(&Version::parse("1.2.3").unwrap())); + Ok(()) + } } From b6d9a162e506a9478725c01983ee5d41f7027166 Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Thu, 26 Sep 2019 11:22:55 -0700 Subject: [PATCH 06/10] updata: Add `add-mapping` command. Include `add-mapping` to manually insert a image-version:datastore-version mapping. This is helpful in particular for adding a mapping for images that may be running but not in the update manifest. Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/updog/src/bin/updata.rs | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs index 5f6b291c730..b961bf782dd 100644 --- a/workspaces/updater/updog/src/bin/updata.rs +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -257,6 +257,35 @@ impl MaxVersionArgs { } } +#[derive(Debug, StructOpt)] +struct MappingArgs { + // metadata file to create/modify + file: PathBuf, + + #[structopt(short, long)] + image_version: Version, + + #[structopt(short, long)] + data_version: DVersion, +} + +impl MappingArgs { + fn run(self) -> Result<()> { + let mut m: Manifest = load_file(&self.file)?; + let version = self.image_version.clone(); + let old = m + .datastore_versions + .insert(self.image_version, self.data_version); + if let Some(old) = old { + eprintln!( + "Warning: New mapping ({},{}) replaced old mapping ({},{})", + version, self.data_version, version, old + ); + } + write_file(&self.file, &m) + } +} + #[derive(Debug, StructOpt)] #[structopt(rename_all = "kebab-case")] enum Command { @@ -264,6 +293,7 @@ enum Command { AddUpdate(AddUpdateArgs), AddWave(WaveArgs), AddMigration(MigrationArgs), + AddMapping(MappingArgs), SetMaxVersion(MaxVersionArgs), RemoveUpdate(RemoveUpdateArgs), RemoveMigration(MigrationArgs), @@ -311,6 +341,7 @@ fn main_inner() -> Result<()> { Command::AddUpdate(args) => args.run(), Command::AddWave(args) => args.add(), Command::AddMigration(args) => args.add(), + Command::AddMapping(args) => args.run(), Command::SetMaxVersion(args) => args.run(), Command::RemoveUpdate(args) => args.run(), Command::RemoveWave(args) => args.remove(), From 4501d6c145d2cbefd185abb41d449279aae9f6ba Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Mon, 21 Oct 2019 16:28:04 -0700 Subject: [PATCH 07/10] update_metadata: Avoid cast in jitter Use checked_sub() instead of casting the difference between the end and start of the wave. This avoids a potential underflow that could cause Updog to jitter itself to a point very far in the future. Fixes #447 Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/update_metadata/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workspaces/updater/update_metadata/src/lib.rs b/workspaces/updater/update_metadata/src/lib.rs index 4f1ce67433c..4afd52ef2d2 100644 --- a/workspaces/updater/update_metadata/src/lib.rs +++ b/workspaces/updater/update_metadata/src/lib.rs @@ -71,14 +71,14 @@ impl Update { let next = self .waves .range((Excluded(seed), Excluded(MAX_SEED))) - .next(); + .next() + .map(|(_, wave)| wave); if let (Some(start), Some(end)) = (prev, next) { - if Utc::now() < *end.1 { + if Utc::now() < *end { let mut rng = thread_rng(); - #[allow(clippy::cast_sign_loss)] - let range = (end.1.timestamp() - start.timestamp()) as u64; - let jitter = Duration::seconds(rng.gen_range(1, range) as i64); - return Some(*start + jitter); + if let Some(range) = end.timestamp().checked_sub(start.timestamp()) { + return Some(*start + Duration::seconds(rng.gen_range(1, range))); + } } } None From efca9f0bb7e6128dab981cddcaa3422aee5d4b9a Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Mon, 21 Oct 2019 16:52:08 -0700 Subject: [PATCH 08/10] updata: Enforce wave ordering Before committing changes to wave metadata check that each successive wave occurs after the previous one. Fixes #447 Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/updog/src/bin/updata.rs | 64 +++++++++++++++++++++- workspaces/updater/updog/src/error.rs | 3 + 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs index b961bf782dd..940f1af9e6c 100644 --- a/workspaces/updater/updog/src/bin/updata.rs +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -7,7 +7,7 @@ use crate::error::Result; use chrono::{DateTime, Utc}; use data_store_version::Version as DVersion; use semver::Version; -use snafu::{ErrorCompat, ResultExt}; +use snafu::{ensure, ErrorCompat, ResultExt}; use std::collections::BTreeMap; use std::fs::{self, File}; use std::path::{Path, PathBuf}; @@ -164,6 +164,24 @@ struct WaveArgs { } impl WaveArgs { + fn validate(updates: &[Update]) -> Result<()> { + for update in updates { + let mut waves = update.waves.iter().peekable(); + while let Some(wave) = waves.next() { + if let Some(next) = waves.peek() { + ensure!( + wave.1 < next.1, + error::WavesUnordered { + wave: *wave.0, + next: *next.0 + } + ); + } + } + } + Ok(()) + } + fn add(self) -> Result<()> { let mut m: Manifest = load_file(&self.file)?; let matching: Vec<&mut Update> = m @@ -178,6 +196,7 @@ impl WaveArgs { for u in matching { u.waves.insert(self.bound, start); } + Self::validate(&m.updates)?; write_file(&self.file, &m) } else { error::WaveStartArg.fail() @@ -373,6 +392,7 @@ fn main() -> ! { #[cfg(test)] mod tests { use super::*; + use chrono::Duration; use std::str::FromStr; use tempfile::NamedTempFile; @@ -486,4 +506,46 @@ mod tests { .contains_key(&Version::parse("1.2.3").unwrap())); Ok(()) } + + #[test] + fn ordered_waves() -> Result<()> { + let tmpfd = NamedTempFile::new().context(error::TmpFileCreate)?; + AddUpdateArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.3").unwrap(), + max_version: Version::parse("1.2.3").unwrap(), + datastore: DVersion::from_str("1.0").unwrap(), + boot: String::from("boot"), + root: String::from("root"), + hash: String::from("hash"), + } + .run() + .unwrap(); + + WaveArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.3").unwrap(), + bound: 1024, + start: Some(Utc::now()), + } + .add() + .unwrap(); + + assert!(WaveArgs { + file: PathBuf::from(tmpfd.path()), + flavor: String::from("yum"), + arch: String::from("x86_64"), + version: Version::parse("1.2.3").unwrap(), + bound: 1536, + start: Some(Utc::now() - Duration::hours(1)), + } + .add() + .is_err()); + + Ok(()) + } } diff --git a/workspaces/updater/updog/src/error.rs b/workspaces/updater/updog/src/error.rs index 2f8cbf341ee..e60354c1dfd 100644 --- a/workspaces/updater/updog/src/error.rs +++ b/workspaces/updater/updog/src/error.rs @@ -260,6 +260,9 @@ pub(crate) enum Error { #[snafu(display("Update wave start time missing"))] WaveStartArg { backtrace: Backtrace }, + #[snafu(display("Waves are not ordered: bound {} occurs before bound {}", next, wave))] + WavesUnordered { wave: u32, next: u32 }, + #[snafu(display("Failed writing update data to disk: {}", source))] WriteUpdate { source: std::io::Error, From 40a013f2c33532bba85a4fb50ed0c29c6d34bdbf Mon Sep 17 00:00:00 2001 From: Samuel Mendoza-Jonas Date: Thu, 24 Oct 2019 15:49:54 -0700 Subject: [PATCH 09/10] updater: Consistent version naming and fixups Update all users of the semver and data_store_version crates to consitently refer to them as "SemVer" and "DataVersion" respectively. Also includes some small fixups to Updata, including: - Allow --max-version to be optional and implicitly set - Display changes to max version - Error if migration marked for removal doesn't exist Signed-off-by: Samuel Mendoza-Jonas --- workspaces/updater/update_metadata/src/de.rs | 26 +- workspaces/updater/update_metadata/src/lib.rs | 12 +- workspaces/updater/update_metadata/src/se.rs | 8 +- workspaces/updater/updog/src/bin/updata.rs | 284 +++++++++++------- workspaces/updater/updog/src/error.rs | 22 +- workspaces/updater/updog/src/main.rs | 60 ++-- 6 files changed, 246 insertions(+), 166 deletions(-) diff --git a/workspaces/updater/update_metadata/src/de.rs b/workspaces/updater/update_metadata/src/de.rs index 2d3e71c0579..98859fa845a 100644 --- a/workspaces/updater/update_metadata/src/de.rs +++ b/workspaces/updater/update_metadata/src/de.rs @@ -1,8 +1,8 @@ use crate::error; use chrono::{DateTime, Utc}; -use data_store_version::Version as DVersion; +use data_store_version::Version as DataVersion; use regex::Regex; -use semver::Version; +use semver::Version as SemVer; use serde::{de::Error as _, Deserializer}; use snafu::{ensure, ResultExt}; use std::collections::BTreeMap; @@ -56,10 +56,10 @@ where deserializer.deserialize_map(Visitor) } -/// Converts the tuple keys to a `DVersion` before insertion and catches duplicates +/// Converts the tuple keys to a `DataVersion` before insertion and catches duplicates pub(crate) fn deserialize_migration<'de, D>( deserializer: D, -) -> Result>, D::Error> +) -> Result>, D::Error> where D: Deserializer<'de>, { @@ -80,11 +80,11 @@ where fn parse_tuple_key( key: String, list: Vec, - map: &mut BTreeMap<(DVersion, DVersion), Vec>, + map: &mut BTreeMap<(DataVersion, DataVersion), Vec>, ) -> Result<(), error::Error> { let (from, to) = parse_versions(&key)?; - if let (Ok(from), Ok(to)) = (DVersion::from_str(from), DVersion::from_str(to)) { + if let (Ok(from), Ok(to)) = (DataVersion::from_str(from), DataVersion::from_str(to)) { ensure!( map.insert((from, to), list).is_none(), error::DuplicateVersionKey { key } @@ -103,7 +103,7 @@ where struct Visitor; impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = BTreeMap<(DVersion, DVersion), Vec>; + type Value = BTreeMap<(DataVersion, DataVersion), Vec>; fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str("a map") @@ -124,21 +124,21 @@ where deserializer.deserialize_map(Visitor) } -/// Converts the key and value into a Version/DVersion pair before insertion and +/// Converts the key and value into a SemVer/DataVersion pair before insertion and /// catches duplicates pub(crate) fn deserialize_datastore_map<'de, D>( deserializer: D, -) -> Result, D::Error> +) -> Result, D::Error> where D: Deserializer<'de>, { fn to_versions( key: String, value: String, - map: &mut BTreeMap, + map: &mut BTreeMap, ) -> Result<(), error::Error> { - let key_ver = Version::parse(&key); - let value_ver = DVersion::from_str(&value); + let key_ver = SemVer::parse(&key); + let value_ver = DataVersion::from_str(&value); match (key_ver, value_ver) { (Ok(k), Ok(v)) => { ensure!( @@ -155,7 +155,7 @@ where struct Visitor; impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = BTreeMap; + type Value = BTreeMap; fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str("a map") diff --git a/workspaces/updater/update_metadata/src/lib.rs b/workspaces/updater/update_metadata/src/lib.rs index 4afd52ef2d2..2a49b6da070 100644 --- a/workspaces/updater/update_metadata/src/lib.rs +++ b/workspaces/updater/update_metadata/src/lib.rs @@ -5,9 +5,9 @@ pub mod error; mod se; use chrono::{DateTime, Duration, Utc}; -use data_store_version::Version as DVersion; +use data_store_version::Version as DataVersion; use rand::{thread_rng, Rng}; -use semver::Version; +use semver::Version as SemVer; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::ops::Bound::{Excluded, Included}; @@ -25,8 +25,8 @@ pub struct Images { pub struct Update { pub flavor: String, pub arch: String, - pub version: Version, - pub max_version: Version, + pub version: SemVer, + pub max_version: SemVer, #[serde(deserialize_with = "de::deserialize_bound")] pub waves: BTreeMap>, pub images: Images, @@ -37,10 +37,10 @@ pub struct Manifest { pub updates: Vec, #[serde(deserialize_with = "de::deserialize_migration")] #[serde(serialize_with = "se::serialize_migration")] - pub migrations: BTreeMap<(DVersion, DVersion), Vec>, + pub migrations: BTreeMap<(DataVersion, DataVersion), Vec>, #[serde(deserialize_with = "de::deserialize_datastore_map")] #[serde(serialize_with = "se::serialize_datastore_map")] - pub datastore_versions: BTreeMap, + pub datastore_versions: BTreeMap, } impl Update { diff --git a/workspaces/updater/update_metadata/src/se.rs b/workspaces/updater/update_metadata/src/se.rs index 7602b485aa2..ccfb7e92bf6 100644 --- a/workspaces/updater/update_metadata/src/se.rs +++ b/workspaces/updater/update_metadata/src/se.rs @@ -1,10 +1,10 @@ -use data_store_version::Version as DVersion; -use semver::Version; +use data_store_version::Version as DataVersion; +use semver::Version as SemVer; use serde::{Serialize, Serializer}; use std::collections::BTreeMap; pub(crate) fn serialize_migration( - value: &BTreeMap<(DVersion, DVersion), Vec>, + value: &BTreeMap<(DataVersion, DataVersion), Vec>, serializer: S, ) -> Result where @@ -23,7 +23,7 @@ where } pub(crate) fn serialize_datastore_map( - value: &BTreeMap, + value: &BTreeMap, serializer: S, ) -> Result where diff --git a/workspaces/updater/updog/src/bin/updata.rs b/workspaces/updater/updog/src/bin/updata.rs index 940f1af9e6c..92a77a77292 100644 --- a/workspaces/updater/updog/src/bin/updata.rs +++ b/workspaces/updater/updog/src/bin/updata.rs @@ -1,13 +1,17 @@ +#![deny(rust_2018_idioms)] #![warn(clippy::pedantic)] #[path = "../error.rs"] mod error; +#[macro_use] +extern crate log; + use crate::error::Result; use chrono::{DateTime, Utc}; -use data_store_version::Version as DVersion; -use semver::Version; -use snafu::{ensure, ErrorCompat, ResultExt}; +use data_store_version::Version as DataVersion; +use semver::Version as SemVer; +use snafu::{ensure, ErrorCompat, OptionExt, ResultExt}; use std::collections::BTreeMap; use std::fs::{self, File}; use std::path::{Path, PathBuf}; @@ -31,7 +35,7 @@ struct AddUpdateArgs { // image version #[structopt(short = "v", long = "version")] - version: Version, + image_version: SemVer, // architecture image is built for #[structopt(short = "a", long = "arch")] @@ -39,11 +43,11 @@ struct AddUpdateArgs { // corresponding datastore version for this image #[structopt(short = "d", long = "data-version")] - datastore: DVersion, + datastore_version: DataVersion, // maximum valid version #[structopt(short = "m", long = "max-version")] - max_version: Version, + max_version: Option, // root image target name #[structopt(short = "r", long = "root")] @@ -60,17 +64,26 @@ struct AddUpdateArgs { impl AddUpdateArgs { fn run(self) -> Result<()> { - let mut m: Manifest = match load_file(&self.file) { + let mut manifest: Manifest = match load_file(&self.file) { Ok(m) => m, _ => Manifest::default(), // TODO only if EEXIST }; - m.datastore_versions - .insert(self.version.clone(), self.datastore); - let u = Update { + + let max_version = if let Some(version) = self.max_version { + version + } else { + // Default to greater of the current max version and this version + if let Some(update) = manifest.updates.first() { + std::cmp::max(&self.image_version, &update.max_version).clone() + } else { + self.image_version.clone() + } + }; + let update = Update { flavor: self.flavor, arch: self.arch, - version: self.version, - max_version: self.max_version, + version: self.image_version.clone(), + max_version, images: Images { root: self.root, boot: self.boot, @@ -78,9 +91,18 @@ impl AddUpdateArgs { }, waves: BTreeMap::new(), }; - update_max_version(&mut m, &u.max_version, Some(&u.arch), Some(&u.flavor)); - m.updates.push(u); - write_file(&self.file, &m) + manifest + .datastore_versions + .insert(self.image_version, self.datastore_version); + update_max_version( + &mut manifest, + &update.max_version, + Some(&update.arch), + Some(&update.flavor), + ); + info!("Maximum version set to {}", &update.max_version); + manifest.updates.push(update); + write_file(&self.file, &manifest) } } @@ -95,7 +117,7 @@ struct RemoveUpdateArgs { // image version #[structopt(short = "v", long = "version")] - version: Version, + image_version: SemVer, // architecture image is built for #[structopt(short = "a", long = "arch")] @@ -112,28 +134,43 @@ struct RemoveUpdateArgs { impl RemoveUpdateArgs { fn run(&self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - m.updates.retain(|u| { - !(u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + let mut manifest: Manifest = load_file(&self.file)?; + // Remove any update that exactly matches the specified update + manifest.updates.retain(|update| { + update.arch != self.arch + || update.flavor != self.flavor + || update.version != self.image_version }); if self.cleanup { - let remaining: Vec<&Update> = m + let remaining: Vec<&Update> = manifest .updates .iter() - .filter(|u| u.version == self.version) + .filter(|update| update.version == self.image_version) .collect(); if remaining.is_empty() { - m.datastore_versions.remove(&self.version); + manifest.datastore_versions.remove(&self.image_version); } else { - println!( + info!( "Cleanup skipped; {} {} updates remain", remaining.len(), - self.version + self.image_version ); } } // Note: We don't revert the maximum version on removal - write_file(&self.file, &m) + write_file(&self.file, &manifest)?; + if let Some(current) = manifest.updates.first() { + info!( + "Update {}-{}-{} removed. Current maximum version: {}", + self.arch, self.flavor, self.image_version, current.version + ); + } else { + info!( + "Update {}-{}-{} removed. No remaining updates", + self.arch, self.flavor, self.image_version + ); + } + Ok(()) } } @@ -148,7 +185,7 @@ struct WaveArgs { // image version #[structopt(short = "v", long = "version")] - version: Version, + image_version: SemVer, // architecture image is built for #[structopt(short = "a", long = "arch")] @@ -183,37 +220,43 @@ impl WaveArgs { } fn add(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - let matching: Vec<&mut Update> = m + let mut manifest: Manifest = load_file(&self.file)?; + let matching: Vec<&mut Update> = manifest .updates .iter_mut() - .filter(|u| u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + // Find the update that exactly matches the specified update + .filter(|update| { + update.arch == self.arch + && update.flavor == self.flavor + && update.version == self.image_version + }) .collect(); if matching.len() > 1 { - println!("Multiple matching updates for wave - this is weird but not a disaster"); + warn!("Multiple matching updates for wave - this is weird but not a disaster"); } - if let Some(start) = self.start { - for u in matching { - u.waves.insert(self.bound, start); - } - Self::validate(&m.updates)?; - write_file(&self.file, &m) - } else { - error::WaveStartArg.fail() + let start = self.start.context(error::WaveStartArg)?; + for update in matching { + update.waves.insert(self.bound, start); } + Self::validate(&manifest.updates)?; + write_file(&self.file, &manifest) } fn remove(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - let matching: Vec<&mut Update> = m + let mut manifest: Manifest = load_file(&self.file)?; + let matching: Vec<&mut Update> = manifest .updates .iter_mut() - .filter(|u| u.arch == self.arch && u.flavor == self.flavor && u.version == self.version) + .filter(|update| { + update.arch == self.arch + && update.flavor == self.flavor + && update.version == self.image_version + }) .collect(); - for u in matching { - u.waves.remove(&self.bound); + for update in matching { + update.waves.remove(&self.bound); } - write_file(&self.file, &m) + write_file(&self.file, &manifest) } } @@ -224,11 +267,11 @@ struct MigrationArgs { // starting datastore version #[structopt(short = "f", long = "from")] - from: DVersion, + from: DataVersion, // target datastore version #[structopt(short = "t", long = "to")] - to: DVersion, + to: DataVersion, // whether to append to or replace any existing migration list #[structopt(short, long)] @@ -240,21 +283,36 @@ struct MigrationArgs { impl MigrationArgs { fn add(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - let mut migrations = self.migrations; - if self.append { - if let Some(e) = m.migrations.remove(&(self.from, self.to)) { - migrations.extend_from_slice(&e); - } + let mut manifest: Manifest = load_file(&self.file)?; + // If --append is set, append the new migrations to the existing vec. + if self.append && manifest.migrations.contains_key(&(self.from, self.to)) { + let migrations = manifest.migrations.get_mut(&(self.from, self.to)).context( + error::MigrationMutable { + from: self.from, + to: self.to, + }, + )?; + migrations.extend_from_slice(&self.migrations); + // Otherwise just overwrite the existing migrations + } else { + manifest + .migrations + .insert((self.from, self.to), self.migrations); } - m.migrations.insert((self.from, self.to), migrations); - write_file(&self.file, &m) + write_file(&self.file, &manifest) } fn remove(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - m.migrations.remove(&(self.from, self.to)); - write_file(&self.file, &m) + let mut manifest: Manifest = load_file(&self.file)?; + ensure!( + manifest.migrations.contains_key(&(self.from, self.to)), + error::MigrationNotPresent { + from: self.from, + to: self.to, + } + ); + manifest.migrations.remove(&(self.from, self.to)); + write_file(&self.file, &manifest) } } @@ -265,14 +323,14 @@ struct MaxVersionArgs { // maximum valid version #[structopt(short, long)] - max_version: Version, + max_version: SemVer, } impl MaxVersionArgs { fn run(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; - update_max_version(&mut m, &self.max_version, None, None); - write_file(&self.file, &m) + let mut manifest: Manifest = load_file(&self.file)?; + update_max_version(&mut manifest, &self.max_version, None, None); + write_file(&self.file, &manifest) } } @@ -282,47 +340,57 @@ struct MappingArgs { file: PathBuf, #[structopt(short, long)] - image_version: Version, + image_version: SemVer, #[structopt(short, long)] - data_version: DVersion, + data_version: DataVersion, } impl MappingArgs { fn run(self) -> Result<()> { - let mut m: Manifest = load_file(&self.file)?; + let mut manifest: Manifest = load_file(&self.file)?; let version = self.image_version.clone(); - let old = m + let old = manifest .datastore_versions .insert(self.image_version, self.data_version); if let Some(old) = old { - eprintln!( + warn!( "Warning: New mapping ({},{}) replaced old mapping ({},{})", version, self.data_version, version, old ); } - write_file(&self.file, &m) + write_file(&self.file, &manifest) } } #[derive(Debug, StructOpt)] #[structopt(rename_all = "kebab-case")] enum Command { + /// Create an empty manifest Init(GeneralArgs), + /// Add a new update to the manifest, not including wave information AddUpdate(AddUpdateArgs), + /// Add a (bound_id, time) wave to an existing update AddWave(WaveArgs), + /// Add one or more migrations to a (from, to) datastore mapping AddMigration(MigrationArgs), - AddMapping(MappingArgs), + /// Add a image_version:data_store_version mapping to the manifest + AddVersionMapping(MappingArgs), + /// Set the global maximum image version SetMaxVersion(MaxVersionArgs), + /// Remove an update from the manifest, including wave information RemoveUpdate(RemoveUpdateArgs), - RemoveMigration(MigrationArgs), + /// Remove all migrations for a (from, to) datastore mapping + RemoveMigrations(MigrationArgs), + /// Remove a (bound_id, time) wave from an update RemoveWave(WaveArgs), + /// Validate a manifest file, but make no changes Validate(GeneralArgs), } fn load_file(path: &Path) -> Result { - serde_json::from_reader(File::open(path).context(error::ManifestRead { path })?) - .context(error::ManifestParse) + let file = File::open(path).context(error::ManifestRead { path })?; + serde_json::from_reader(file).context(error::ManifestParse) } fn write_file(path: &Path, manifest: &Manifest) -> Result<()> { @@ -334,18 +402,18 @@ fn write_file(path: &Path, manifest: &Manifest) -> Result<()> { /// Update the maximum version for all updates that optionally match the /// architecture and flavor of some new update. fn update_max_version( - m: &mut Manifest, - version: &Version, + manifest: &mut Manifest, + version: &SemVer, arch: Option<&str>, flavor: Option<&str>, ) { - let matching: Vec<&mut Update> = m + let matching: Vec<&mut Update> = manifest .updates .iter_mut() - .filter(|u| match (arch, flavor) { - (Some(arch), Some(flavor)) => u.arch == arch && u.flavor == flavor, - (Some(arch), None) => u.arch == arch, - (None, Some(flavor)) => u.flavor == flavor, + .filter(|update| match (arch, flavor) { + (Some(arch), Some(flavor)) => update.arch == arch && update.flavor == flavor, + (Some(arch), None) => update.arch == arch, + (None, Some(flavor)) => update.flavor == flavor, _ => true, }) .collect(); @@ -360,11 +428,11 @@ fn main_inner() -> Result<()> { Command::AddUpdate(args) => args.run(), Command::AddWave(args) => args.add(), Command::AddMigration(args) => args.add(), - Command::AddMapping(args) => args.run(), + Command::AddVersionMapping(args) => args.run(), Command::SetMaxVersion(args) => args.run(), Command::RemoveUpdate(args) => args.run(), Command::RemoveWave(args) => args.remove(), - Command::RemoveMigration(args) => args.remove(), + Command::RemoveMigrations(args) => args.remove(), Command::Validate(args) => match load_file(&args.file) { Ok(_) => Ok(()), Err(e) => Err(e), @@ -376,11 +444,11 @@ fn main() -> ! { std::process::exit(match main_inner() { Ok(()) => 0, Err(err) => { - eprintln!("{}", err); + error!("{}", err); if let Some(var) = std::env::var_os("RUST_BACKTRACE") { if var != "0" { if let Some(backtrace) = err.backtrace() { - eprintln!("\n{:?}", backtrace); + error!("\n{:?}", backtrace); } } } @@ -403,9 +471,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.3").unwrap(), - max_version: Version::parse("1.2.3").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.3").unwrap(), + max_version: Some(SemVer::parse("1.2.3").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -416,9 +484,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.5").unwrap(), - max_version: Version::parse("1.2.3").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.5").unwrap(), + max_version: Some(SemVer::parse("1.2.3").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -429,9 +497,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.4").unwrap(), - max_version: Version::parse("1.2.4").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.4").unwrap(), + max_version: Some(SemVer::parse("1.2.4").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -441,7 +509,7 @@ mod tests { let m: Manifest = load_file(tmpfd.path())?; for u in m.updates { - assert!(u.max_version == Version::parse("1.2.4").unwrap()); + assert!(u.max_version == SemVer::parse("1.2.4").unwrap()); } Ok(()) } @@ -453,9 +521,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.3").unwrap(), - max_version: Version::parse("1.2.3").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.3").unwrap(), + max_version: Some(SemVer::parse("1.2.3").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -466,9 +534,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.5").unwrap(), - max_version: Version::parse("1.2.3").unwrap(), - datastore: DVersion::from_str("1.1").unwrap(), + image_version: SemVer::parse("1.2.5").unwrap(), + max_version: Some(SemVer::parse("1.2.3").unwrap()), + datastore_version: DataVersion::from_str("1.1").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -479,9 +547,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.4").unwrap(), - max_version: Version::parse("1.2.4").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.4").unwrap(), + max_version: Some(SemVer::parse("1.2.4").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -494,7 +562,7 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.4").unwrap(), + image_version: SemVer::parse("1.2.4").unwrap(), cleanup: true, } .run() @@ -503,7 +571,7 @@ mod tests { let m: Manifest = load_file(tmpfd.path())?; assert!(m .datastore_versions - .contains_key(&Version::parse("1.2.3").unwrap())); + .contains_key(&SemVer::parse("1.2.3").unwrap())); Ok(()) } @@ -514,9 +582,9 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.3").unwrap(), - max_version: Version::parse("1.2.3").unwrap(), - datastore: DVersion::from_str("1.0").unwrap(), + image_version: SemVer::parse("1.2.3").unwrap(), + max_version: Some(SemVer::parse("1.2.3").unwrap()), + datastore_version: DataVersion::from_str("1.0").unwrap(), boot: String::from("boot"), root: String::from("root"), hash: String::from("hash"), @@ -528,7 +596,7 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.3").unwrap(), + image_version: SemVer::parse("1.2.3").unwrap(), bound: 1024, start: Some(Utc::now()), } @@ -539,7 +607,7 @@ mod tests { file: PathBuf::from(tmpfd.path()), flavor: String::from("yum"), arch: String::from("x86_64"), - version: Version::parse("1.2.3").unwrap(), + image_version: SemVer::parse("1.2.3").unwrap(), bound: 1536, start: Some(Utc::now() - Duration::hours(1)), } diff --git a/workspaces/updater/updog/src/error.rs b/workspaces/updater/updog/src/error.rs index e60354c1dfd..b2eb88e1fa4 100644 --- a/workspaces/updater/updog/src/error.rs +++ b/workspaces/updater/updog/src/error.rs @@ -140,6 +140,20 @@ pub(crate) enum Error { #[snafu(display("Migration not found in image: {:?}", name))] MigrationNotLocal { backtrace: Backtrace, name: PathBuf }, + #[snafu(display("Unable to get mutable reference to ({},{}) migrations", from, to))] + MigrationMutable { + backtrace: Backtrace, + from: DataVersion, + to: DataVersion, + }, + + #[snafu(display("Migration ({},{}) not present in manifest", from, to))] + MigrationNotPresent { + backtrace: Backtrace, + from: DataVersion, + to: DataVersion, + }, + #[snafu(display("Missing datastore version in metadata: {:?}", version))] MissingDataVersion { backtrace: Backtrace, @@ -251,13 +265,7 @@ pub(crate) enum Error { backtrace: Backtrace, }, - #[snafu(display("Update wave start time missing"))] - WaveStartArg { backtrace: Backtrace }, - - #[snafu(display("Waves are not ordered: bound {} occurs before bound {}", next, wave))] - WavesUnordered { wave: u32, next: u32 }, - - #[snafu(display("Update wave start time missing"))] + #[snafu(display("--start-time