From 49019d0ee40d1d78fbfc80f240e274964fd26869 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 31 Jul 2023 15:05:29 +0200 Subject: [PATCH 01/12] adapt for vector upgrade --- tests/templates/kuttl/logging/test_log_aggregation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index d2b28848..a66f552a 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -12,8 +12,8 @@ def check_processed_events(): nodes { componentId metrics { - processedEventsTotal { - processedEventsTotal + receivedEventsTotal { + receivedEventsTotal } } } @@ -30,9 +30,9 @@ def check_processed_events(): transforms = result['data']['transforms']['nodes'] for transform in transforms: - processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal'] + receivedEvents = transform['metrics']['receivedEventsTotal']['receivedEventsTotal'] componentId = transform['componentId'] - assert processedEvents > 0, \ + assert receivedEvents > 0, \ f'No events were processed in "{componentId}".' From 5761e5ff9363e633f4d75ad156cf81491de0c6da Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 31 Jul 2023 15:19:27 +0200 Subject: [PATCH 02/12] patch operator-rs --- Cargo.lock | 4 ++-- Cargo.toml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4974c46e..8c6bc964 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1895,7 +1895,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.44.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.44.0#f922369d05e709c0965098012d0fd76865004982" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=upgrade-vector#618b9a316e465dd247a5b0c85cde9a46d2305328" dependencies = [ "chrono", "clap", @@ -1929,7 +1929,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.44.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.44.0#f922369d05e709c0965098012d0fd76865004982" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=upgrade-vector#618b9a316e465dd247a5b0c85cde9a46d2305328" dependencies = [ "darling 0.20.1", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index ecff712e..6efa656b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,6 @@ members = [ #[patch."https://github.com/stackabletech/operator-rs.git"] #stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } + +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "upgrade-vector"} \ No newline at end of file From 82a91b5fc4d5201e285ac3b6e75f521bfd6f1800 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 31 Jul 2023 15:38:45 +0200 Subject: [PATCH 03/12] changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d0d30e..a69e0ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Changed + +- `vector` `0.26.0` -> `0.31.0` ([#465]). +- `operator-rs` `0.44.0` -> `0.45.0` ([#465]). + +[#465]: https://github.com/stackabletech/opa-operator/pull/465 + ## [23.7.0] - 2023-07-14 ### Added From fc776c43db6cbf9039607269b0f192fa80eff4ec Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 1 Aug 2023 16:57:24 +0200 Subject: [PATCH 04/12] bump vector aggregator --- .../kuttl/logging/01-install-opa-vector-aggregator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml index 3912b573..b8e68aa5 100644 --- a/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml @@ -5,7 +5,7 @@ commands: - script: >- helm install opa-vector-aggregator vector --namespace $NAMESPACE - --version 0.19.0 + --version 0.23.0 --repo https://helm.vector.dev --values opa-vector-aggregator-values.yaml --- From 5b057185162d37081b1569705ceba3cb6526ee94 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 1 Aug 2023 17:01:24 +0200 Subject: [PATCH 05/12] bump operator-rs --- Cargo.lock | 38 ++++++++++++++++++++++++++------- Cargo.toml | 3 --- rust/crd/Cargo.toml | 2 +- rust/operator-binary/Cargo.toml | 6 +++--- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8c6bc964..fc95fd0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1868,7 +1868,7 @@ dependencies = [ "serde_json", "snafu", "stackable-operator", - "strum", + "strum 0.24.1", "tracing", ] @@ -1887,15 +1887,15 @@ dependencies = [ "snafu", "stackable-opa-crd", "stackable-operator", - "strum", + "strum 0.25.0", "tokio", "tracing", ] [[package]] name = "stackable-operator" -version = "0.44.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=upgrade-vector#618b9a316e465dd247a5b0c85cde9a46d2305328" +version = "0.45.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.45.1#94141b9cd2142050e8c976d98c56c568dc85a5bd" dependencies = [ "chrono", "clap", @@ -1918,7 +1918,7 @@ dependencies = [ "serde_yaml 0.9.22", "snafu", "stackable-operator-derive", - "strum", + "strum 0.24.1", "thiserror", "tokio", "tracing", @@ -1928,8 +1928,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.44.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=upgrade-vector#618b9a316e465dd247a5b0c85cde9a46d2305328" +version = "0.45.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.45.1#94141b9cd2142050e8c976d98c56c568dc85a5bd" dependencies = [ "darling 0.20.1", "proc-macro2", @@ -1949,7 +1949,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros 0.25.1", ] [[package]] @@ -1965,6 +1974,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.22", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 6efa656b..ecff712e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,3 @@ members = [ #[patch."https://github.com/stackabletech/operator-rs.git"] #stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } - -[patch."https://github.com/stackabletech/operator-rs.git"] -stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "upgrade-vector"} \ No newline at end of file diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 0347a88c..2e1776a0 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -9,7 +9,7 @@ version = "0.0.0-dev" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.44.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.45.1" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 7f359267..c00059c9 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -10,7 +10,7 @@ version = "0.0.0-dev" publish = false [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.44.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.45.1" } stackable-opa-crd = { path = "../crd" } clap = "4.1" @@ -20,12 +20,12 @@ semver = "1.0" serde = "1.0" serde_json = "1.0" snafu = "0.7" -strum = { version = "0.24", features = ["derive"] } +strum = { version = "0.25", features = ["derive"] } tokio = { version = "1.28", features = ["full"] } tracing = "0.1" pin-project = "1.0" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.44.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.45.1" } stackable-opa-crd = { path = "../crd" } From eb866e53189df73c1bdd4abc9f07239d138c4cc4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 1 Aug 2023 17:01:48 +0200 Subject: [PATCH 06/12] stackable version --- CHANGELOG.md | 6 +++++- deploy/helm/opa-operator/crds/crds.yaml | 6 +++--- rust/operator-binary/src/controller.rs | 5 ++++- rust/operator-binary/src/main.rs | 1 + 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69e0ac8..cbd2eed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,14 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- Default stackableVersion to operator version ([#465]). + ### Changed - `vector` `0.26.0` -> `0.31.0` ([#465]). -- `operator-rs` `0.44.0` -> `0.45.0` ([#465]). +- `operator-rs` `0.44.0` -> `0.45.1` ([#465]). [#465]: https://github.com/stackabletech/opa-operator/pull/465 diff --git a/deploy/helm/opa-operator/crds/crds.yaml b/deploy/helm/opa-operator/crds/crds.yaml index 527bb73b..c69a9979 100644 --- a/deploy/helm/opa-operator/crds/crds.yaml +++ b/deploy/helm/opa-operator/crds/crds.yaml @@ -71,7 +71,6 @@ spec: - productVersion - required: - productVersion - - stackableVersion description: The OPA image to use properties: custom: @@ -81,7 +80,7 @@ spec: description: Version of the product, e.g. `1.4.1`. type: string pullPolicy: - default: IfNotPresent + default: Always description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the Images' enum: - IfNotPresent @@ -104,7 +103,8 @@ spec: nullable: true type: string stackableVersion: - description: Stackable version of the product, e.g. 2.1.0 + description: Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + nullable: true type: string type: object servers: diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 85e880ad..f513ea08 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -216,7 +216,10 @@ pub async fn reconcile_opa(opa: Arc, ctx: Arc) -> Result = option_env!("TARGET"); + pub const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); } #[derive(Parser)] From f1636b3e09c32e5fbfeb4bc993912ae1ebef68c8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 1 Aug 2023 17:05:46 +0200 Subject: [PATCH 07/12] switch tests to sentEventsTotal --- .../opa-vector-aggregator-values.yaml.j2 | 8 +++---- .../kuttl/logging/test_log_aggregation.py | 23 ++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 index df58523d..8f21eb9b 100644 --- a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 @@ -20,25 +20,25 @@ customConfig: type: vector version: "2" transforms: - automaticLogConfigServerOpa: + filteredAutomaticLogConfigServerOpa: type: filter inputs: [vector] condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "opa" - automaticLogConfigServerBundleBuilder: + filteredAutomaticLogConfigServerBundleBuilder: type: filter inputs: [vector] condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "bundle-builder" - automaticLogConfigServerVector: + filteredAutomaticLogConfigServerVector: type: filter inputs: [vector] condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "vector" - automaticLogConfigServerPrepare: + filteredAutomaticLogConfigServerPrepare: type: filter inputs: [vector] condition: >- diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index a66f552a..b2a9e39f 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -2,7 +2,7 @@ import requests -def check_processed_events(): +def check_sent_events(): response = requests.post( 'http://opa-vector-aggregator:8686/graphql', json={ @@ -12,8 +12,8 @@ def check_processed_events(): nodes { componentId metrics { - receivedEventsTotal { - receivedEventsTotal + sentEventsTotal { + sentEventsTotal } } } @@ -30,12 +30,19 @@ def check_processed_events(): transforms = result['data']['transforms']['nodes'] for transform in transforms: - receivedEvents = transform['metrics']['receivedEventsTotal']['receivedEventsTotal'] + sentEvents = transform['metrics']['sentEventsTotal'] componentId = transform['componentId'] - assert receivedEvents > 0, \ - f'No events were processed in "{componentId}".' + + if componentId == 'filteredInvalidEvents': + assert sentEvents is None or \ + sentEvents['sentEventsTotal'] == 0, \ + 'Invalid log events were sent.' + else: + assert sentEvents is not None and \ + sentEvents['sentEventsTotal'] > 0, \ + f'No events were sent in "{componentId}".' if __name__ == '__main__': - check_processed_events() - print('Test successful!') + check_sent_events() + print('Test successful!') \ No newline at end of file From b63d9dcf062df0aedbdb7f0027200f0ee90a44d5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 1 Aug 2023 17:06:34 +0200 Subject: [PATCH 08/12] new line --- tests/templates/kuttl/logging/test_log_aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index b2a9e39f..21a8020a 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -45,4 +45,4 @@ def check_sent_events(): if __name__ == '__main__': check_sent_events() - print('Test successful!') \ No newline at end of file + print('Test successful!') From 83651a6d46d7400562fd6f66a371be0082f94fe5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 2 Aug 2023 09:26:35 +0200 Subject: [PATCH 09/12] add filtered invalid events --- .../logging/opa-vector-aggregator-values.yaml.j2 | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 index 8f21eb9b..5bb70ed7 100644 --- a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 @@ -44,12 +44,24 @@ customConfig: condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "prepare" + filteredInvalidEvents: + type: filter + inputs: [vector] + condition: |- + .timestamp == from_unix_timestamp!(0) || + is_null(.level) || + is_null(.logger) || + is_null(.message) sinks: out: - inputs: [automaticLogConfig*] + inputs: [filtered*] {% if lookup('env', 'VECTOR_AGGREGATOR') %} type: vector address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} + buffer: + # Avoid back pressure from VECTOR_AGGREGATOR. The test should + # not fail if the aggregator is not available. + when_full: drop_newest {% else %} type: blackhole {% endif %} From 2d5d1983798e3344b725abda971570317658e495 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 2 Aug 2023 09:44:02 +0200 Subject: [PATCH 10/12] attempt to fix linter --- tests/templates/kuttl/logging/test_log_aggregation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py index 21a8020a..b5d50dc5 100755 --- a/tests/templates/kuttl/logging/test_log_aggregation.py +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -36,11 +36,11 @@ def check_sent_events(): if componentId == 'filteredInvalidEvents': assert sentEvents is None or \ sentEvents['sentEventsTotal'] == 0, \ - 'Invalid log events were sent.' + 'Invalid log events were sent.' else: assert sentEvents is not None and \ sentEvents['sentEventsTotal'] > 0, \ - f'No events were sent in "{componentId}".' + f'No events were sent in "{componentId}".' if __name__ == '__main__': From a19334c3292b8b25e958fe1822d4dbbed8dc376f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 2 Aug 2023 15:00:52 +0200 Subject: [PATCH 11/12] adapt kuttl test --- tests/kuttl-test.yaml.jinja2 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2 index 76b502aa..9d0b5efb 100644 --- a/tests/kuttl-test.yaml.jinja2 +++ b/tests/kuttl-test.yaml.jinja2 @@ -9,3 +9,19 @@ testDirs: startKIND: false suppress: ["events"] parallel: 2 + +# The timeout (in seconds) is used when namespaces are created or +# deleted, and, if not overridden, in TestSteps, TestAsserts, and +# Commands. If not set, the timeout is 30 seconds by default. +# +# The deletion of a namespace can take a while until all resources are +# gracefully shut down. If the timeout is reached in the meantime, even +# a successful test case is considered a failure. +# +# For instance, the termination grace period of the Vector aggregator in +# the logging tests is set to 60 seconds. If there are logs entries +# which could not be forwarded yet to the external aggregator defined in +# the VECTOR_AGGREGATOR environment variable, then the test aggregator +# uses this period of time by trying to forward the events. In this +# case, deleting a namespace with several Pods takes about 90 seconds. +timeout: 120 From 800e76045605234db81d6b17794048b64cce3bff Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 2 Aug 2023 15:10:56 +0200 Subject: [PATCH 12/12] align versions --- Cargo.lock | 30 ++++-------------------------- rust/operator-binary/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26267e66..1f7e1095 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1867,7 +1867,7 @@ dependencies = [ "serde_json", "snafu", "stackable-operator", - "strum 0.24.1", + "strum", "tracing", ] @@ -1886,7 +1886,7 @@ dependencies = [ "snafu", "stackable-opa-crd", "stackable-operator", - "strum 0.25.0", + "strum", "tokio", "tracing", ] @@ -1917,7 +1917,7 @@ dependencies = [ "serde_yaml 0.9.25", "snafu", "stackable-operator-derive", - "strum 0.24.1", + "strum", "thiserror", "tokio", "tracing", @@ -1948,16 +1948,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros 0.24.3", -] - -[[package]] -name = "strum" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" -dependencies = [ - "strum_macros 0.25.1", + "strum_macros", ] [[package]] @@ -1973,19 +1964,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "strum_macros" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.22", -] - [[package]] name = "syn" version = "1.0.109" diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 6204ef3f..125deac5 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -20,7 +20,7 @@ semver = "1.0" serde = "1.0" serde_json = "1.0" snafu = "0.7" -strum = { version = "0.25", features = ["derive"] } +strum = { version = "0.24", features = ["derive"] } tokio = { version = "1.28", features = ["full"] } tracing = "0.1" pin-project = "1.0"