feat(configuration): make config url overridable via env var (#218)

* Add mockito for http mocking

* Add helper fn for testing with http mock

* *Settings::new now accepts CONFIG_URI to load a remote config

* Make *Settings::new async

* Add test for CONFIG_URI, fix existing test

* Update CHANGELOGs

* Rename CONFIG_URI CONFIG_URL

* Make remove env handling from run_test_with_http_response so it is nestable with run_test_with_env

* Fix test and add new assert

* Update CHANGELOGs

agents/kathy/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Unreleased
 
+- make *Settings::new async for optionally fetching config from a remote url
+- add test for remote config fetch
 - add bootup-only tracing subscriber
 - add environment variable overrides for agent configuration
 - add tests for agent environment variable overrides

agents/kathy/src/main.rs

@@ -18,6 +18,7 @@ use tracing_subscriber::prelude::*;
 async fn main() -> Result<()> {
     color_eyre::install()?;
 
+    // sets the subscriber for this scope only
     let _bootup_guard = tracing_subscriber::FmtSubscriber::builder()
         .json()
         .with_level(true)
@@ -26,7 +27,7 @@ async fn main() -> Result<()> {
     let span = info_span!("KathyBootup");
     let _span = span.enter();
 
-    let settings = Settings::new()?;
+    let settings = Settings::new().await?;
     let agent = Kathy::from_settings(settings).await?;
 
     drop(_span);
@@ -36,5 +37,6 @@ async fn main() -> Result<()> {
 
     let _ = agent.metrics().run_http_server();
 
-    agent.run_all().await?
+    agent.run_all().await??;
+    Ok(())
 }

agents/kathy/src/settings.rs

@@ -11,15 +11,67 @@ mod test {
     use nomad_base::{get_remotes_from_env, NomadAgent};
     use nomad_test::test_utils;
     use nomad_xyz_configuration::{agent::kathy::ChatGenConfig, AgentSecrets};
+    use serde_json::json;
+    use std::collections::HashSet;
 
-    #[test]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn it_loads_remote_config() {
+        let config =
+            nomad_xyz_configuration::NomadConfig::from_file("../../fixtures/external_config.json")
+                .unwrap();
+        let config_json = json!(config).to_string();
+        test_utils::run_test_with_http_response(config_json, |url| async move {
+            test_utils::run_test_with_env("../../fixtures/env.external", || async move {
+                std::env::set_var("CONFIG_URL", url);
+
+                let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
+
+                let settings = KathySettings::new().await.unwrap();
+
+                let remotes = get_remotes_from_env!(agent_home, config);
+                let mut networks = remotes.clone();
+                networks.insert(agent_home.clone());
+
+                let secrets = AgentSecrets::from_env(&networks).unwrap();
+                secrets
+                    .validate("kathy", &networks)
+                    .expect("!secrets validate");
+
+                settings
+                    .base
+                    .validate_against_config_and_secrets(
+                        crate::Kathy::AGENT_NAME,
+                        &agent_home,
+                        &remotes,
+                        &config,
+                        &secrets,
+                    )
+                    .unwrap();
+
+                let mut settings_networks = settings
+                    .base
+                    .replicas
+                    .values()
+                    .map(|c| c.name.clone())
+                    .collect::<HashSet<_>>();
+                settings_networks.insert(settings.base.home.name.clone());
+
+                assert_eq!(config.networks, settings_networks);
+            })
+            .await
+        })
+        .await
+    }
+
+    #[tokio::test]
     #[serial_test::serial]
-    fn it_overrides_settings_from_env() {
-        test_utils::run_test_with_env_sync("../../fixtures/env.test-agents", move || {
+    async fn it_overrides_settings_from_env() {
+        test_utils::run_test_with_env("../../fixtures/env.test-agents", || async move {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = KathySettings::new().unwrap();
+            let settings = KathySettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 
@@ -50,15 +102,16 @@ mod test {
                 }
             );
             assert_eq!(settings.agent.interval, 999);
-        });
+        })
+        .await
     }
 
     async fn test_build_from_env_file(path: &str) {
         test_utils::run_test_with_env(path, || async move {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = KathySettings::new().unwrap();
+            let settings = KathySettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 
@@ -108,7 +161,7 @@ mod test {
             std::env::set_var("CONFIG_PATH", "../../fixtures/external_config.json");
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = KathySettings::new().unwrap();
+            let settings = KathySettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::NomadConfig::from_file(
                 "../../fixtures/external_config.json",

agents/processor/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- make *Settings::new async for optionally fetching config from a remote url
 - add bootup-only tracing subscriber
 - bug: add check for empty intersection of specified and subsidized
 - refactor: processor now uses global AWS client when proof pushing is enabled

agents/processor/src/main.rs

@@ -25,6 +25,7 @@ use tracing_subscriber::prelude::*;
 async fn main() -> Result<()> {
     color_eyre::install()?;
 
+    // sets the subscriber for this scope only
     let _bootup_guard = tracing_subscriber::FmtSubscriber::builder()
         .json()
         .with_level(true)
@@ -33,7 +34,7 @@ async fn main() -> Result<()> {
     let span = info_span!("ProcessorBootup");
     let _span = span.enter();
 
-    let settings = Settings::new()?;
+    let settings = Settings::new().await?;
     let agent = Processor::from_settings(settings).await?;
 
     drop(_span);

agents/processor/src/settings.rs

@@ -19,7 +19,7 @@ mod test {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = ProcessorSettings::new().unwrap();
+            let settings = ProcessorSettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 

agents/relayer/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- make *Settings::new async for optionally fetching config from a remote url
 - relayer checks replica updater addresses match, errors channel if otherwise
 - add bootup-only tracing subscriber
 - add environment variable overrides for agent configuration

agents/relayer/src/main.rs

@@ -21,6 +21,7 @@ use tracing_subscriber::prelude::*;
 async fn main() -> Result<()> {
     color_eyre::install()?;
 
+    // sets the subscriber for this scope only
     let _bootup_guard = tracing_subscriber::FmtSubscriber::builder()
         .json()
         .with_level(true)
@@ -29,7 +30,7 @@ async fn main() -> Result<()> {
     let span = info_span!("RelayerBootup");
     let _span = span.enter();
 
-    let settings = Settings::new()?;
+    let settings = Settings::new().await?;
     let agent = Relayer::from_settings(settings).await?;
 
     drop(_span);

agents/relayer/src/settings.rs

@@ -19,7 +19,7 @@ mod test {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = RelayerSettings::new().unwrap();
+            let settings = RelayerSettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 

agents/updater/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- make *Settings::new async for optionally fetching config from a remote url
 - add bootup-only tracing subscriber
 - add environment variable overrides for agent configuration
 - add tests for agent environment variable overrides

agents/updater/src/main.rs

@@ -22,6 +22,7 @@ use tracing_subscriber::prelude::*;
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<()> {
     color_eyre::install()?;
+
     // sets the subscriber for this scope only
     let _bootup_guard = tracing_subscriber::FmtSubscriber::builder()
         .json()
@@ -31,7 +32,7 @@ async fn main() -> Result<()> {
     let span = info_span!("UpdaterBootup");
     let _span = span.enter();
 
-    let settings = Settings::new()?;
+    let settings = Settings::new().await?;
     let agent = Updater::from_settings(settings).await?;
 
     drop(_span);

agents/updater/src/settings.rs

@@ -18,7 +18,7 @@ mod test {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = UpdaterSettings::new().unwrap();
+            let settings = UpdaterSettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 

agents/watcher/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- make *Settings::new async for optionally fetching config from a remote url
 - add bootup-only tracing subscriber
 - add environment variable overrides for agent configuration
 - add tests for agent environment variable overrides

agents/watcher/src/main.rs

@@ -32,13 +32,14 @@ async fn main() -> Result<()> {
     let span = info_span!("WatcherBootup");
     let _span = span.enter();
 
-    let settings = Settings::new()?;
+    let settings = Settings::new().await?;
     let agent = Watcher::from_settings(settings).await?;
 
     drop(_span);
     drop(span);
 
     let _tracing_guard = agent.start_tracing(agent.metrics().span_duration());
+
     let _ = agent.metrics().run_http_server();
 
     agent.run_all().await??;

agents/watcher/src/settings.rs

@@ -19,7 +19,7 @@ mod test {
             let run_env = dotenv::var("RUN_ENV").unwrap();
             let agent_home = dotenv::var("AGENT_HOME_NAME").unwrap();
 
-            let settings = WatcherSettings::new().unwrap();
+            let settings = WatcherSettings::new().await.unwrap();
 
             let config = nomad_xyz_configuration::get_builtin(&run_env).unwrap();
 

nomad-base/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Unreleased
 
+- add `CONFIG_URL` check to `decl_settings` to optionally fetch config from a remote url
 - bug: add checks for empty replica name arrays in `NomadAgent::run_many` and
   `NomadAgent::run_all`
 - add `previously_attempted` to the DB schema

nomad-base/src/settings/macros.rs

@@ -72,19 +72,25 @@ macro_rules! decl_settings {
             }
 
             impl [<$name Settings>] {
-                pub fn new() -> color_eyre::Result<Self>{
+                pub async fn new() -> color_eyre::Result<Self>{
                     // Get agent and home names
                     tracing::info!("Building settings from env");
                     let agent = std::stringify!($name).to_lowercase();
                     let home = std::env::var("AGENT_HOME_NAME").expect("missing AGENT_HOME_NAME");
 
                     // Get config
-                    let config_path = std::env::var("CONFIG_PATH").ok();
-                    let config: nomad_xyz_configuration::NomadConfig = match config_path {
-                        Some(path) => nomad_xyz_configuration::NomadConfig::from_file(path).expect("!config"),
+                    let config_url = std::env::var("CONFIG_URL").ok();
+                    let config: nomad_xyz_configuration::NomadConfig = match config_url {
+                        Some(url) => nomad_xyz_configuration::NomadConfig::fetch(&url).await.expect("!config url"),
                         None => {
-                            let env = std::env::var("RUN_ENV").expect("missing RUN_ENV");
-                            nomad_xyz_configuration::get_builtin(&env).expect("!config").to_owned()
+                            let config_path = std::env::var("CONFIG_PATH").ok();
+                            match config_path {
+                                Some(path) => nomad_xyz_configuration::NomadConfig::from_file(path).expect("!config path"),
+                                None => {
+                                    let env = std::env::var("RUN_ENV").expect("missing RUN_ENV");
+                                    nomad_xyz_configuration::get_builtin(&env).expect("!config builtin").to_owned()
+                                }
+                            }
                         }
                     };
                     config.validate()?;

nomad-test/CHANGELOG.md

@@ -2,4 +2,6 @@
 
 ### Unreleased
 
+- add helper fn for testing with an http mock response
+- add mockito for mocking http responses
 - adds a changelog

nomad-test/Cargo.toml

@@ -20,6 +20,7 @@ rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb" }
 dotenv = "0.15.0"
 tracing = "0.1.35"
 prometheus = "0.12.0"
+mockito = "0.31.0"
 
 nomad-xyz-configuration = { path = "../configuration" }
 nomad-core = { path = "../nomad-core" }