From 7740c335c4da6e070fcb3ff839a6025414079486 Mon Sep 17 00:00:00 2001 From: Matthew Wiriyathananon-Smith Date: Thu, 21 Jul 2022 21:11:10 +0700 Subject: [PATCH] feat(configuration): make config url overridable via env var (#218) * Add mockito for http mocking * Add helper fn for testing with http mock * *Settings::new now accepts CONFIG_URI to load a remote config * Make *Settings::new async * Add test for CONFIG_URI, fix existing test * Update CHANGELOGs * Rename CONFIG_URI CONFIG_URL * Make remove env handling from run_test_with_http_response so it is nestable with run_test_with_env * Fix test and add new assert * Update CHANGELOGs --- Cargo.lock | 35 ++++++++++++++++ agents/kathy/CHANGELOG.md | 2 + agents/kathy/src/main.rs | 6 ++- agents/kathy/src/settings.rs | 67 +++++++++++++++++++++++++++---- agents/processor/CHANGELOG.md | 1 + agents/processor/src/main.rs | 3 +- agents/processor/src/settings.rs | 2 +- agents/relayer/CHANGELOG.md | 1 + agents/relayer/src/main.rs | 3 +- agents/relayer/src/settings.rs | 2 +- agents/updater/CHANGELOG.md | 1 + agents/updater/src/main.rs | 3 +- agents/updater/src/settings.rs | 2 +- agents/watcher/CHANGELOG.md | 1 + agents/watcher/src/main.rs | 3 +- agents/watcher/src/settings.rs | 2 +- nomad-base/CHANGELOG.md | 1 + nomad-base/src/settings/macros.rs | 18 ++++++--- nomad-test/CHANGELOG.md | 2 + nomad-test/Cargo.toml | 1 + nomad-test/src/test_utils.rs | 21 ++++++++++ 21 files changed, 154 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b294ff6d..6ff7aa42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,16 @@ dependencies = [ "term", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde 1.0.137", + "serde_json", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -2385,6 +2395,24 @@ dependencies = [ "syn", ] +[[package]] +name = "mockito" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401edc088069634afaa5f4a29617b36dba683c0c16fe4435a86debad23fa2f1a" +dependencies = [ + "assert-json-diff", + "colored", + "httparse", + "lazy_static", + "log", + "rand", + "regex", + "serde_json", + "serde_urlencoded", + "similar", +] + [[package]] name = "multipart" version = "0.18.0" @@ -2575,6 +2603,7 @@ dependencies = [ "ethers", "futures-util", "mockall 0.10.2", + "mockito", "nomad-core", "nomad-ethereum", "nomad-xyz-configuration", @@ -4094,6 +4123,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "similar" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e24979f63a11545f5f2c60141afe249d4f19f84581ea2138065e400941d83d3" + [[package]] name = "siphasher" version = "0.3.10" diff --git a/agents/kathy/CHANGELOG.md b/agents/kathy/CHANGELOG.md index f583de4f..ea3c5a96 100644 --- a/agents/kathy/CHANGELOG.md +++ b/agents/kathy/CHANGELOG.md @@ -2,6 +2,8 @@ ### Unreleased +- make *Settings::new async for optionally fetching config from a remote url +- add test for remote config fetch - add bootup-only tracing subscriber - add environment variable overrides for agent configuration - add tests for agent environment variable overrides diff --git a/agents/kathy/src/main.rs b/agents/kathy/src/main.rs index 01df879f..91870f12 100644 --- a/agents/kathy/src/main.rs +++ b/agents/kathy/src/main.rs @@ -18,6 +18,7 @@ use tracing_subscriber::prelude::*; async fn main() -> Result<()> { color_eyre::install()?; + // sets the subscriber for this scope only let _bootup_guard = tracing_subscriber::FmtSubscriber::builder() .json() .with_level(true) @@ -26,7 +27,7 @@ async fn main() -> Result<()> { let span = info_span!("KathyBootup"); let _span = span.enter(); - let settings = Settings::new()?; + let settings = Settings::new().await?; let agent = Kathy::from_settings(settings).await?; drop(_span); @@ -36,5 +37,6 @@ async fn main() -> Result<()> { let _ = agent.metrics().run_http_server(); - agent.run_all().await? + agent.run_all().await??; + Ok(()) } diff --git a/agents/kathy/src/settings.rs b/agents/kathy/src/settings.rs index 4690a279..8e6c81fe 100644 --- a/agents/kathy/src/settings.rs +++ b/agents/kathy/src/settings.rs @@ -11,15 +11,67 @@ mod test { use nomad_base::{get_remotes_from_env, NomadAgent}; use nomad_test::test_utils; use nomad_xyz_configuration::{agent::kathy::ChatGenConfig, AgentSecrets}; + use serde_json::json; + use std::collections::HashSet; - #[test] + #[tokio::test] + #[serial_test::serial] + async fn it_loads_remote_config() { + let config = + nomad_xyz_configuration::NomadConfig::from_file("../../fixtures/external_config.json") + .unwrap(); + let config_json = json!(config).to_string(); + test_utils::run_test_with_http_response(config_json, |url| async move { + test_utils::run_test_with_env("../../fixtures/env.external", || async move { + std::env::set_var("CONFIG_URL", url); + + let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); + + let settings = KathySettings::new().await.unwrap(); + + let remotes = get_remotes_from_env!(agent_home, config); + let mut networks = remotes.clone(); + networks.insert(agent_home.clone()); + + let secrets = AgentSecrets::from_env(&networks).unwrap(); + secrets + .validate("kathy", &networks) + .expect("!secrets validate"); + + settings + .base + .validate_against_config_and_secrets( + crate::Kathy::AGENT_NAME, + &agent_home, + &remotes, + &config, + &secrets, + ) + .unwrap(); + + let mut settings_networks = settings + .base + .replicas + .values() + .map(|c| c.name.clone()) + .collect::>(); + settings_networks.insert(settings.base.home.name.clone()); + + assert_eq!(config.networks, settings_networks); + }) + .await + }) + .await + } + + #[tokio::test] #[serial_test::serial] - fn it_overrides_settings_from_env() { - test_utils::run_test_with_env_sync("../../fixtures/env.test-agents", move || { + async fn it_overrides_settings_from_env() { + test_utils::run_test_with_env("../../fixtures/env.test-agents", || async move { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = KathySettings::new().unwrap(); + let settings = KathySettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); @@ -50,7 +102,8 @@ mod test { } ); assert_eq!(settings.agent.interval, 999); - }); + }) + .await } async fn test_build_from_env_file(path: &str) { @@ -58,7 +111,7 @@ mod test { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = KathySettings::new().unwrap(); + let settings = KathySettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); @@ -108,7 +161,7 @@ mod test { std::env::set_var("CONFIG_PATH", "../../fixtures/external_config.json"); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = KathySettings::new().unwrap(); + let settings = KathySettings::new().await.unwrap(); let config = nomad_xyz_configuration::NomadConfig::from_file( "../../fixtures/external_config.json", diff --git a/agents/processor/CHANGELOG.md b/agents/processor/CHANGELOG.md index bfb54e71..e0953e54 100644 --- a/agents/processor/CHANGELOG.md +++ b/agents/processor/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- make *Settings::new async for optionally fetching config from a remote url - add bootup-only tracing subscriber - bug: add check for empty intersection of specified and subsidized - refactor: processor now uses global AWS client when proof pushing is enabled diff --git a/agents/processor/src/main.rs b/agents/processor/src/main.rs index f5df68a1..598dd7e4 100644 --- a/agents/processor/src/main.rs +++ b/agents/processor/src/main.rs @@ -25,6 +25,7 @@ use tracing_subscriber::prelude::*; async fn main() -> Result<()> { color_eyre::install()?; + // sets the subscriber for this scope only let _bootup_guard = tracing_subscriber::FmtSubscriber::builder() .json() .with_level(true) @@ -33,7 +34,7 @@ async fn main() -> Result<()> { let span = info_span!("ProcessorBootup"); let _span = span.enter(); - let settings = Settings::new()?; + let settings = Settings::new().await?; let agent = Processor::from_settings(settings).await?; drop(_span); diff --git a/agents/processor/src/settings.rs b/agents/processor/src/settings.rs index 7a695c9f..c8342c7f 100644 --- a/agents/processor/src/settings.rs +++ b/agents/processor/src/settings.rs @@ -19,7 +19,7 @@ mod test { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = ProcessorSettings::new().unwrap(); + let settings = ProcessorSettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); diff --git a/agents/relayer/CHANGELOG.md b/agents/relayer/CHANGELOG.md index c361c7a8..bc28b55f 100644 --- a/agents/relayer/CHANGELOG.md +++ b/agents/relayer/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- make *Settings::new async for optionally fetching config from a remote url - relayer checks replica updater addresses match, errors channel if otherwise - add bootup-only tracing subscriber - add environment variable overrides for agent configuration diff --git a/agents/relayer/src/main.rs b/agents/relayer/src/main.rs index 34fd92b9..2617f3b4 100644 --- a/agents/relayer/src/main.rs +++ b/agents/relayer/src/main.rs @@ -21,6 +21,7 @@ use tracing_subscriber::prelude::*; async fn main() -> Result<()> { color_eyre::install()?; + // sets the subscriber for this scope only let _bootup_guard = tracing_subscriber::FmtSubscriber::builder() .json() .with_level(true) @@ -29,7 +30,7 @@ async fn main() -> Result<()> { let span = info_span!("RelayerBootup"); let _span = span.enter(); - let settings = Settings::new()?; + let settings = Settings::new().await?; let agent = Relayer::from_settings(settings).await?; drop(_span); diff --git a/agents/relayer/src/settings.rs b/agents/relayer/src/settings.rs index c3777a9b..e1f0a207 100644 --- a/agents/relayer/src/settings.rs +++ b/agents/relayer/src/settings.rs @@ -19,7 +19,7 @@ mod test { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = RelayerSettings::new().unwrap(); + let settings = RelayerSettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); diff --git a/agents/updater/CHANGELOG.md b/agents/updater/CHANGELOG.md index f583de4f..1a5c72e0 100644 --- a/agents/updater/CHANGELOG.md +++ b/agents/updater/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- make *Settings::new async for optionally fetching config from a remote url - add bootup-only tracing subscriber - add environment variable overrides for agent configuration - add tests for agent environment variable overrides diff --git a/agents/updater/src/main.rs b/agents/updater/src/main.rs index ac625127..7313367c 100644 --- a/agents/updater/src/main.rs +++ b/agents/updater/src/main.rs @@ -22,6 +22,7 @@ use tracing_subscriber::prelude::*; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { color_eyre::install()?; + // sets the subscriber for this scope only let _bootup_guard = tracing_subscriber::FmtSubscriber::builder() .json() @@ -31,7 +32,7 @@ async fn main() -> Result<()> { let span = info_span!("UpdaterBootup"); let _span = span.enter(); - let settings = Settings::new()?; + let settings = Settings::new().await?; let agent = Updater::from_settings(settings).await?; drop(_span); diff --git a/agents/updater/src/settings.rs b/agents/updater/src/settings.rs index 3d2d7a7c..84718ac4 100644 --- a/agents/updater/src/settings.rs +++ b/agents/updater/src/settings.rs @@ -18,7 +18,7 @@ mod test { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = UpdaterSettings::new().unwrap(); + let settings = UpdaterSettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); diff --git a/agents/watcher/CHANGELOG.md b/agents/watcher/CHANGELOG.md index f583de4f..1a5c72e0 100644 --- a/agents/watcher/CHANGELOG.md +++ b/agents/watcher/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- make *Settings::new async for optionally fetching config from a remote url - add bootup-only tracing subscriber - add environment variable overrides for agent configuration - add tests for agent environment variable overrides diff --git a/agents/watcher/src/main.rs b/agents/watcher/src/main.rs index 253b7555..09cfdf2d 100644 --- a/agents/watcher/src/main.rs +++ b/agents/watcher/src/main.rs @@ -32,13 +32,14 @@ async fn main() -> Result<()> { let span = info_span!("WatcherBootup"); let _span = span.enter(); - let settings = Settings::new()?; + let settings = Settings::new().await?; let agent = Watcher::from_settings(settings).await?; drop(_span); drop(span); let _tracing_guard = agent.start_tracing(agent.metrics().span_duration()); + let _ = agent.metrics().run_http_server(); agent.run_all().await??; diff --git a/agents/watcher/src/settings.rs b/agents/watcher/src/settings.rs index 27ecb453..3aa55105 100644 --- a/agents/watcher/src/settings.rs +++ b/agents/watcher/src/settings.rs @@ -19,7 +19,7 @@ mod test { let run_env = dotenv::var("RUN_ENV").unwrap(); let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap(); - let settings = WatcherSettings::new().unwrap(); + let settings = WatcherSettings::new().await.unwrap(); let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap(); diff --git a/nomad-base/CHANGELOG.md b/nomad-base/CHANGELOG.md index aec72fc0..c93429f2 100644 --- a/nomad-base/CHANGELOG.md +++ b/nomad-base/CHANGELOG.md @@ -2,6 +2,7 @@ ### Unreleased +- add `CONFIG_URL` check to `decl_settings` to optionally fetch config from a remote url - bug: add checks for empty replica name arrays in `NomadAgent::run_many` and `NomadAgent::run_all` - add `previously_attempted` to the DB schema diff --git a/nomad-base/src/settings/macros.rs b/nomad-base/src/settings/macros.rs index b12b2378..e3d5cd34 100644 --- a/nomad-base/src/settings/macros.rs +++ b/nomad-base/src/settings/macros.rs @@ -72,19 +72,25 @@ macro_rules! decl_settings { } impl [<$name Settings>] { - pub fn new() -> color_eyre::Result{ + pub async fn new() -> color_eyre::Result{ // Get agent and home names tracing::info!("Building settings from env"); let agent = std::stringify!($name).to_lowercase(); let home = std::env::var("AGENT_HOME_NAME").expect("missing AGENT_HOME_NAME"); // Get config - let config_path = std::env::var("CONFIG_PATH").ok(); - let config: nomad_xyz_configuration::NomadConfig = match config_path { - Some(path) => nomad_xyz_configuration::NomadConfig::from_file(path).expect("!config"), + let config_url = std::env::var("CONFIG_URL").ok(); + let config: nomad_xyz_configuration::NomadConfig = match config_url { + Some(url) => nomad_xyz_configuration::NomadConfig::fetch(&url).await.expect("!config url"), None => { - let env = std::env::var("RUN_ENV").expect("missing RUN_ENV"); - nomad_xyz_configuration::get_builtin(&env).expect("!config").to_owned() + let config_path = std::env::var("CONFIG_PATH").ok(); + match config_path { + Some(path) => nomad_xyz_configuration::NomadConfig::from_file(path).expect("!config path"), + None => { + let env = std::env::var("RUN_ENV").expect("missing RUN_ENV"); + nomad_xyz_configuration::get_builtin(&env).expect("!config builtin").to_owned() + } + } } }; config.validate()?; diff --git a/nomad-test/CHANGELOG.md b/nomad-test/CHANGELOG.md index 1d10aca9..c3c04e93 100644 --- a/nomad-test/CHANGELOG.md +++ b/nomad-test/CHANGELOG.md @@ -2,4 +2,6 @@ ### Unreleased +- add helper fn for testing with an http mock response +- add mockito for mocking http responses - adds a changelog diff --git a/nomad-test/Cargo.toml b/nomad-test/Cargo.toml index 66a0c0c8..27a95010 100644 --- a/nomad-test/Cargo.toml +++ b/nomad-test/Cargo.toml @@ -20,6 +20,7 @@ rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb" } dotenv = "0.15.0" tracing = "0.1.35" prometheus = "0.12.0" +mockito = "0.31.0" nomad-xyz-configuration = { path = "../configuration" } nomad-core = { path = "../nomad-core" } diff --git a/nomad-test/src/test_utils.rs b/nomad-test/src/test_utils.rs index 77291da1..24adb638 100644 --- a/nomad-test/src/test_utils.rs +++ b/nomad-test/src/test_utils.rs @@ -1,4 +1,5 @@ use futures_util::FutureExt; +use mockito; use nomad_core::db::DB; use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; @@ -68,6 +69,26 @@ where assert!(result.is_ok()) } +/// Run test with a mock http server response +pub async fn run_test_with_http_response(response_body: impl AsRef<[u8]>, test: T) +where + T: FnOnce(String) -> Fut + panic::UnwindSafe, + Fut: Future, +{ + let result = { + let url = mockito::server_url(); + let _m = mockito::mock("GET", "/") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(response_body) + .create(); + let func = panic::AssertUnwindSafe(async { test(url).await }); + func.catch_unwind().await + }; + + assert!(result.is_ok()) +} + pub fn clear_env_vars() { let env_vars = env::vars(); for (key, _) in env_vars {