fix: Fixing Cohort project state (#98)

* fix: Fixing Cohort project state
* fix: Set toolchain version to 1.73.0

.github/workflows/build.yml

@@ -8,9 +8,9 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v3
-            - uses: dtolnay/rust-toolchain@stable
+            - uses: dtolnay/rust-toolchain@1.73.0
               with:
-                  toolchain: stable
+                  toolchain: 1.73.0
             - run: scripts/ubuntu-setup.sh
             - run: rustup component add rustfmt clippy
             - name: cargo build

.github/workflows/coverage.yml

@@ -8,9 +8,9 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/checkout@v3
-            - uses: dtolnay/rust-toolchain@nightly
+            - uses: dtolnay/rust-toolchain@master
               with:
-                  toolchain: nightly
+                  toolchain: nightly-2023-10-03
             - run: scripts/ubuntu-setup.sh
             - run: rustup component add llvm-tools-preview
             - run: curl -L https://github.com/mozilla/grcov/releases/latest/download/grcov-x86_64-unknown-linux-gnu.tar.bz2 | tar jxf -

Makefile

@@ -165,14 +165,12 @@ cohort_banking.run_replicator_rust:
 ## cohort_banking.run_initiator_js: 🧪 Executes load test through Cohort Initiator implemented in JS
 cohort_banking.run_initiator_load_test_in_js:
 	$(call pp,running "Cohort Initiator" implemented in JS...)
-	cd ./cohort_banking_initiator_js
-	npm start -- $(args)
+	cd ./cohort_banking_initiator_js && npm start -- $(args)
 
 ## cohort_banking.run_replicator_js: 🧪 Runs Replicator JS app
 cohort_banking.run_replicator_js:
 	$(call pp,running "Cohort Replicator" implemented in JS...)
-	cd ./cohort_banking_replicator_js
-	npm start
+	cd ./cohort_banking_replicator_js && npm start
 
 ## dev.histogram_decision_timeline_from_kafka: 🧪 Reads all decisions from kafka and prints processing timeline as csv
 dev.histogram_decision_timeline_from_kafka:

cohort_banking_initiator_js/src/cfg/config-cohort-sdk.ts

@@ -1,5 +1,11 @@
 import { JsInitiatorConfig } from "@kindredgroup/cohort_sdk_client"
 
+const getParam = (param: string, defaultValue: string): string => {
+    const v = process.env[param]
+    if (v) return v
+    return defaultValue
+}
+
 const SDK_CONFIG: JsInitiatorConfig = {
     backoffOnConflict: {
         minMs: 2,
@@ -22,7 +28,7 @@ const SDK_CONFIG: JsInitiatorConfig = {
       timeoutMs: 60_000,
       kafka: {
         brokers: ["127.0.0.1:9092"],
-        topic: "dev.ksp.certification",
+        topic: getParam("KAFKA_TOPIC", "dev.ksp.certification"),
         clientId: "cohort-js",
         groupId: "cohort-js",
         producerSendTimeoutMs: 10,
@@ -31,8 +37,8 @@ const SDK_CONFIG: JsInitiatorConfig = {
         consumerConfigOverrides: {
             "enable.auto.commit": "false"
         },
-        username: "",
-        password: "",
+        username: getParam("KAFKA_USERNAME", ""),
+        password: getParam("KAFKA_PASSWORD", ""),
       },
 }
 

cohort_banking_initiator_js/src/cfg/config-db-pool.ts

@@ -1,15 +1,21 @@
 import { PoolConfig } from "pg"
 
+const getParam = (param: string, defaultValue: string): string => {
+    const v = process.env[param]
+    if (v) return v
+    return defaultValue
+}
+
 const DB_CONFIG: PoolConfig = {
     application_name: "cohort_banking_js",
-    keepAlive: true,
-    host: "127.0.0.1",
-    port: 5432,
-    database: "talos-sample-cohort-dev",
-    user: "postgres",
-    password: "admin",
-    max: 100,
-    min: 100,
+    keepAlive:  true,
+    host:       getParam("COHORT_PG_HOST", "127.0.0.1"),
+    port:       parseInt(getParam("COHORT_PG_PORT", "5432")),
+    database:   getParam("COHORT_PG_DATABASE", "talos-sample-cohort-dev"),
+    user:       getParam("COHORT_PG_USER", "postgres"),
+    password:   getParam("COHORT_PG_PASSWORD", "admin"),
+    max:        parseInt(getParam("COHORT_PG_POOL_SIZE", "100")),
+    min:        parseInt(getParam("COHORT_PG_POOL_SIZE", "100")),
 }
 
 export { DB_CONFIG }

cohort_banking_replicator_js/package.json

@@ -19,7 +19,7 @@
     "typescript": "^5.1.6"
   },
   "dependencies": {
-    "@kindredgroup/cohort_sdk_client": "^0.0.1-0ed84cd",
+    "@kindredgroup/cohort_sdk_client": "^0.2.10",
     "pg": "^8.11.3",
     "winston": "^3.10.0"
   }

cohort_banking_replicator_js/src/cfg/config-db-pool.ts

@@ -1,15 +1,21 @@
 import { PoolConfig } from "pg"
 
+const getParam = (param: string, defaultValue: string): string => {
+    const v = process.env[param]
+    if (v) return v
+    return defaultValue
+}
+
 const DB_CONFIG: PoolConfig = {
     application_name: "cohort_banking_replicator_js",
-    keepAlive: true,
-    host: "127.0.0.1",
-    port: 5432,
-    database: "talos-sample-cohort-dev",
-    user: "postgres",
-    password: "admin",
-    max: 100,
-    min: 100,
+    keepAlive:  true,
+    host:       getParam("COHORT_PG_HOST", "127.0.0.1"),
+    port:       parseInt(getParam("COHORT_PG_PORT", "5432")),
+    database:   getParam("COHORT_PG_DATABASE", "talos-sample-cohort-dev"),
+    user:       getParam("COHORT_PG_USER", "postgres"),
+    password:   getParam("COHORT_PG_PASSWORD", "admin"),
+    max:        parseInt(getParam("COHORT_PG_POOL_SIZE", "100")),
+    min:        parseInt(getParam("COHORT_PG_POOL_SIZE", "100")),
 }
 
 export { DB_CONFIG }

cohort_banking_replicator_js/src/cfg/config-kafka.ts

@@ -1,16 +1,22 @@
 import { JsKafkaConfig } from "@kindredgroup/cohort_sdk_client"
 
+const getParam = (param: string, defaultValue: string): string => {
+    const v = process.env[param]
+    if (v) return v
+    return defaultValue
+}
+
 const kafkaConfig: JsKafkaConfig = {
     brokers: ["127.0.0.1:9092"],
-    topic: "dev.ksp.certification",
+    topic: getParam("KAFKA_TOPIC", "dev.ksp.certification"),
     clientId: "cohort-replicator-js",
     groupId: "cohort-replicator-js",
     producerSendTimeoutMs: 10,
     logLevel: "info",
     producerConfigOverrides: {},
     consumerConfigOverrides: { "enable.auto.commit": "false" },
-    username: "",
-    password: "",
+    username: getParam("KAFKA_USERNAME", ""),
+    password: getParam("KAFKA_PASSWORD", ""),
 }
 
 export { kafkaConfig as KAFKA_CONFIG }

examples/cohort_banking_with_sdk/examples/cohort_banking_with_sdk.rs

@@ -35,8 +35,8 @@ struct LaunchParams {
     metric_print_raw: bool,
 }
 
-/// Connects to database, to kafka certification topic as talso agent and as cohort replicator.
-/// Generates some number of banking transactions and passes then all to Talos for certification.
+/// Connects to database, to kafka certification topic as talos agent and as cohort replicator.
+/// Generates some number of banking transactions and passes them all to Talos for certification.
 /// Once all transactions have been processed, it prints some output metrics to console.
 /// Metric logging is set to WARN. If RUST_LOG is set to stricter than WARN then no metrics will be printed.
 /// Preprequisite:
@@ -110,14 +110,16 @@ async fn main() -> Result<(), String> {
         },
     };
 
-    let db_config = DatabaseConfig {
-        pool_size: 100,
-        user: "postgres".into(),
-        password: "admin".into(),
-        host: "127.0.0.1".into(),
-        port: "5432".into(),
-        database: "talos-sample-cohort-dev".into(),
-    };
+    let db_config = DatabaseConfig::from_env(Some("COHORT"))?;
+
+    // let db_config = DatabaseConfig {
+    //     pool_size: 100,
+    //     user: "postgres".into(),
+    //     password: "admin".into(),
+    //     host: "127.0.0.1".into(),
+    //     port: "5432".into(),
+    //     database: "talos-sample-cohort-dev".into(),
+    // };
 
     let printer = MetricsToStringPrinter::new(params.threads, params.metric_print_raw, ScalingConfig { ratios: params.scaling_config });
     let (tx_metrics, rx_metrics) = tokio::sync::watch::channel("".to_string());

packages/cohort_sdk_js/README.md

@@ -6,7 +6,7 @@ When publishing it is being packaged as __incomplete__ NPM module and published
 
 The content of NPM module published to repository is missing `index.js`, `index.d.ts` and _native library_ with SDK implementation. All these missig files are generated by [NAPI-RS](https://napi.rs/) during **package install time**. 
 
-When `cohort_sdk_js` NPM package is installed via `npm -i` the "postinstall" hook is invoking `cohort_sdk_js/scripts/postinstall.sh` which in turn compiles Rust code for your architecture, then generates JS code and moves files to their indended destinations so that we end up with finished working NPM package. All this happens on the client machine under `./node_modules/@kindredgroup/cohort_sdk_js`. The build process is verbose if installation is invoked with additional flag: `npm -i @kindredgroup/cohort_sdk_js@SOME_VERSION --forground-scripts`, otherwise native compilation happens in the background.
+When `cohort_sdk_js` NPM package is installed via `npm -i` the "postinstall" hook is invoking `cohort_sdk_js/scripts/postinstall.sh` which in turn compiles Rust code for your architecture, then generates JS code and moves files to their indended destinations so that we end up with finished working NPM package. All this happens on the client machine under `./node_modules/@kindredgroup/cohort_sdk_js`. The build process is verbose if installation is invoked with additional flag: `npm -i @kindredgroup/cohort_sdk_js@SOME_VERSION --foreground-scripts`, otherwise native compilation happens in the background.
 
 As such, for prerequisites of installing this package we require that Rust development environment is setup on the user machine.
 

packages/cohort_sdk_js/scripts/bundle-talos.sh

@@ -17,11 +17,11 @@ cp -r \
     ../talos_certifier_adapters \
     ../talos_cohort_replicator \
     ../talos_common_utils \
+    ../talos_metrics \
     ../talos_rdkafka_utils \
     ../talos_suffix \
     ../cohort_sdk \
     ../logger \
-    ../metrics \
     $TARGET_DIR/packages/
 
 cp -r \

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "stable" #1.68
+channel = "1.73.0" #1.68

scripts/cohort/data-generator.py

@@ -1,4 +1,4 @@
-#!python3
+#!/opt/homebrew/bin/python3
 
 import sys
 import math

scripts/coverage-report.sh

@@ -16,11 +16,11 @@ export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Co
 excludes="cohort_sdk_js"
 
 echo "Compiling $app_name"
-cargo +nightly build
+cargo +nightly-2023-10-03 build
 
 echo "Testing $app_name"
 export LLVM_PROFILE_FILE="${app_name}-%p-%m.profraw"
-cargo +nightly test --tests --workspace --exclude ${excludes} # don't run doctests
+cargo +nightly-2023-10-03 test --tests --workspace --exclude ${excludes} # don't run doctests
 
 rm ccov.zip 2> /dev/null || true
 zip -0 ccov.zip `find . \( -name "*.gc*" \) -print | grep -v ${excludes}`
@@ -38,7 +38,7 @@ grcov ccov.zip -s . --llvm  --ignore-not-existing --ignore "/*" --excl-start "\\
 rm ccov.zip
 
 # Re-run tests with JSON output
-cargo +nightly test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json
+cargo +nightly-2023-10-03 test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json
 
 if [ "$1" == "--open" ]; then
   index="file://$(pwd)/${base_dir}/../coverage/index.html"