From 861d738041198c0ca395b7e884fc9a8a81b4aa30 Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Mon, 19 Jun 2023 11:17:29 -0700 Subject: [PATCH 1/6] SearchBackPressure Collector added (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- licenses/guava-31.1-jre.jar.sha1 | 1 - licenses/guava-32.0.1-jre.jar.sha1 | 1 + licenses/j2objc-annotations-1.3.jar.sha1 | 1 - licenses/j2objc-annotations-2.8.jar.sha1 | 1 + licenses/jackson-annotations-2.15.1.jar.sha1 | 1 - licenses/jackson-annotations-2.15.2.jar.sha1 | 1 + licenses/jackson-databind-2.15.1.jar.sha1 | 1 - licenses/jackson-databind-2.15.2.jar.sha1 | 1 + .../jackson-module-paranamer-2.15.1.jar.sha1 | 1 - .../jackson-module-paranamer-2.15.2.jar.sha1 | 1 + licenses/netty-buffer-4.1.91.Final.jar.sha1 | 1 - licenses/netty-buffer-4.1.93.Final.jar.sha1 | 1 + licenses/netty-codec-4.1.91.Final.jar.sha1 | 1 - licenses/netty-codec-4.1.93.Final.jar.sha1 | 1 + .../netty-codec-http-4.1.91.Final.jar.sha1 | 1 - .../netty-codec-http-4.1.93.Final.jar.sha1 | 1 + .../netty-codec-http2-4.1.91.Final.jar.sha1 | 1 - .../netty-codec-http2-4.1.93.Final.jar.sha1 | 1 + .../netty-codec-socks-4.1.91.Final.jar.sha1 | 1 - .../netty-codec-socks-4.1.93.Final.jar.sha1 | 1 + licenses/netty-common-4.1.91.Final.jar.sha1 | 1 - licenses/netty-common-4.1.93.Final.jar.sha1 | 1 + licenses/netty-handler-4.1.91.Final.jar.sha1 | 1 - licenses/netty-handler-4.1.93.Final.jar.sha1 | 1 + .../netty-handler-proxy-4.1.91.Final.jar.sha1 | 1 - .../netty-handler-proxy-4.1.93.Final.jar.sha1 | 1 + licenses/netty-resolver-4.1.91.Final.jar.sha1 | 1 - licenses/netty-resolver-4.1.93.Final.jar.sha1 | 1 + .../netty-transport-4.1.91.Final.jar.sha1 | 1 - .../netty-transport-4.1.93.Final.jar.sha1 | 1 + ...t-native-unix-common-4.1.91.Final.jar.sha1 | 1 - ...t-native-unix-common-4.1.93.Final.jar.sha1 | 1 + ...e-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 | 2 +- .../PerformanceAnalyzerPlugin.java | 5 + .../SearchBackPressureStatsCollector.java | 665 ++++++++++++++++++ .../performanceanalyzer/util/Utils.java | 1 + ...SearchBackPressureStatsCollectorTests.java | 169 +++++ 37 files changed, 857 insertions(+), 17 deletions(-) delete mode 100644 licenses/guava-31.1-jre.jar.sha1 create mode 100644 licenses/guava-32.0.1-jre.jar.sha1 delete mode 100644 licenses/j2objc-annotations-1.3.jar.sha1 create mode 100644 licenses/j2objc-annotations-2.8.jar.sha1 delete mode 100644 licenses/jackson-annotations-2.15.1.jar.sha1 create mode 100644 licenses/jackson-annotations-2.15.2.jar.sha1 delete mode 100644 licenses/jackson-databind-2.15.1.jar.sha1 create mode 100644 licenses/jackson-databind-2.15.2.jar.sha1 delete mode 100644 licenses/jackson-module-paranamer-2.15.1.jar.sha1 create mode 100644 licenses/jackson-module-paranamer-2.15.2.jar.sha1 delete mode 100644 licenses/netty-buffer-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-buffer-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-http-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-http-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-http2-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-http2-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-socks-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-socks-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-common-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-common-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-handler-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-handler-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-resolver-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-resolver-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-transport-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-transport-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 create mode 100644 src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java create mode 100644 src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java diff --git a/licenses/guava-31.1-jre.jar.sha1 b/licenses/guava-31.1-jre.jar.sha1 deleted file mode 100644 index e57390eb..00000000 --- a/licenses/guava-31.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -60458f877d055d0c9114d9e1a2efb737b4bc282c \ No newline at end of file diff --git a/licenses/guava-32.0.1-jre.jar.sha1 b/licenses/guava-32.0.1-jre.jar.sha1 new file mode 100644 index 00000000..80dc9e93 --- /dev/null +++ b/licenses/guava-32.0.1-jre.jar.sha1 @@ -0,0 +1 @@ +6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/licenses/j2objc-annotations-1.3.jar.sha1 b/licenses/j2objc-annotations-1.3.jar.sha1 deleted file mode 100644 index 0332207a..00000000 --- a/licenses/j2objc-annotations-1.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ba035118bc8bac37d7eff77700720999acd9986d \ No newline at end of file diff --git a/licenses/j2objc-annotations-2.8.jar.sha1 b/licenses/j2objc-annotations-2.8.jar.sha1 new file mode 100644 index 00000000..dd70876c --- /dev/null +++ b/licenses/j2objc-annotations-2.8.jar.sha1 @@ -0,0 +1 @@ +c85270e307e7b822f1086b93689124b89768e273 \ No newline at end of file diff --git a/licenses/jackson-annotations-2.15.1.jar.sha1 b/licenses/jackson-annotations-2.15.1.jar.sha1 deleted file mode 100644 index a3ade4ff..00000000 --- a/licenses/jackson-annotations-2.15.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -092a90d3739e970e03b5971839e4fe51f13c1fa3 \ No newline at end of file diff --git a/licenses/jackson-annotations-2.15.2.jar.sha1 b/licenses/jackson-annotations-2.15.2.jar.sha1 new file mode 100644 index 00000000..f63416dd --- /dev/null +++ b/licenses/jackson-annotations-2.15.2.jar.sha1 @@ -0,0 +1 @@ +4724a65ac8e8d156a24898d50fd5dbd3642870b8 \ No newline at end of file diff --git a/licenses/jackson-databind-2.15.1.jar.sha1 b/licenses/jackson-databind-2.15.1.jar.sha1 deleted file mode 100644 index 47405b86..00000000 --- a/licenses/jackson-databind-2.15.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ac9ba74d208faf356e4719a49e59c6ea9237c01d \ No newline at end of file diff --git a/licenses/jackson-databind-2.15.2.jar.sha1 b/licenses/jackson-databind-2.15.2.jar.sha1 new file mode 100644 index 00000000..f16d80af --- /dev/null +++ b/licenses/jackson-databind-2.15.2.jar.sha1 @@ -0,0 +1 @@ +9353b021f10c307c00328f52090de2bdb4b6ff9c \ No newline at end of file diff --git a/licenses/jackson-module-paranamer-2.15.1.jar.sha1 b/licenses/jackson-module-paranamer-2.15.1.jar.sha1 deleted file mode 100644 index 50c1d67d..00000000 --- a/licenses/jackson-module-paranamer-2.15.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1eee39f6a814b12d92a936cd198657ff26970215 \ No newline at end of file diff --git a/licenses/jackson-module-paranamer-2.15.2.jar.sha1 b/licenses/jackson-module-paranamer-2.15.2.jar.sha1 new file mode 100644 index 00000000..11724827 --- /dev/null +++ b/licenses/jackson-module-paranamer-2.15.2.jar.sha1 @@ -0,0 +1 @@ +afa71e1b9e9eee5664481cab589487d0a7f83ff4 \ No newline at end of file diff --git a/licenses/netty-buffer-4.1.91.Final.jar.sha1 b/licenses/netty-buffer-4.1.91.Final.jar.sha1 deleted file mode 100644 index 158024bc..00000000 --- a/licenses/netty-buffer-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8f180291c3501e931968ca7e40ae0323c4eacee \ No newline at end of file diff --git a/licenses/netty-buffer-4.1.93.Final.jar.sha1 b/licenses/netty-buffer-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..5c5a17a9 --- /dev/null +++ b/licenses/netty-buffer-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +87fac21f4ef95157866b07b48e3c707a2f13c581 \ No newline at end of file diff --git a/licenses/netty-codec-4.1.91.Final.jar.sha1 b/licenses/netty-codec-4.1.91.Final.jar.sha1 deleted file mode 100644 index 764a03d3..00000000 --- a/licenses/netty-codec-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3044b8e325e33f72c96ac1ea51dda85bef090cc0 \ No newline at end of file diff --git a/licenses/netty-codec-4.1.93.Final.jar.sha1 b/licenses/netty-codec-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..2b12a111 --- /dev/null +++ b/licenses/netty-codec-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +503badb458b6586632be8d1f81aa4e5ab99a80fc \ No newline at end of file diff --git a/licenses/netty-codec-http-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http-4.1.91.Final.jar.sha1 deleted file mode 100644 index ca956129..00000000 --- a/licenses/netty-codec-http-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4519d2ff470941f0086214b19c9acf992868112f \ No newline at end of file diff --git a/licenses/netty-codec-http-4.1.93.Final.jar.sha1 b/licenses/netty-codec-http-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..6719e882 --- /dev/null +++ b/licenses/netty-codec-http-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +36acf0c94d03eb6ecef78a749a32cbb7dc0c57b4 \ No newline at end of file diff --git a/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 deleted file mode 100644 index d57336af..00000000 --- a/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ee7027e1653c6ee3f843191e0d932f29e8e14e1 \ No newline at end of file diff --git a/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..02423842 --- /dev/null +++ b/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +0f1625b43bde13ec057da0d2fe381ded2547a70e \ No newline at end of file diff --git a/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 deleted file mode 100644 index 10d7478c..00000000 --- a/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f0a52677da411a8ab762c426d723c7f54471504 \ No newline at end of file diff --git a/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..3e3f699a --- /dev/null +++ b/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +81ca78969afc60073e47c3b7b361cc3839392c73 \ No newline at end of file diff --git a/licenses/netty-common-4.1.91.Final.jar.sha1 b/licenses/netty-common-4.1.91.Final.jar.sha1 deleted file mode 100644 index deaad405..00000000 --- a/licenses/netty-common-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -93e5056462a242718e7689d81180d125c79d7723 \ No newline at end of file diff --git a/licenses/netty-common-4.1.93.Final.jar.sha1 b/licenses/netty-common-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..2324a54d --- /dev/null +++ b/licenses/netty-common-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +1cfc49b91b0d3ddb30c9f7d8467e5d02ae8babdf \ No newline at end of file diff --git a/licenses/netty-handler-4.1.91.Final.jar.sha1 b/licenses/netty-handler-4.1.91.Final.jar.sha1 deleted file mode 100644 index 3e121e5d..00000000 --- a/licenses/netty-handler-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -444cf41e4fe28c47ffebba5e77b9458a12f938a1 \ No newline at end of file diff --git a/licenses/netty-handler-4.1.93.Final.jar.sha1 b/licenses/netty-handler-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..54e5b220 --- /dev/null +++ b/licenses/netty-handler-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +10f7ed9d8e1bfcba416074c70e5388be96116bfc \ No newline at end of file diff --git a/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 deleted file mode 100644 index 116ed58f..00000000 --- a/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2e3e57eae1a61e4e5f558e39619186fec6c424d3 \ No newline at end of file diff --git a/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..1ac94e65 --- /dev/null +++ b/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +c165c1efe1b9c0cc22546a057b530611a088768b \ No newline at end of file diff --git a/licenses/netty-resolver-4.1.91.Final.jar.sha1 b/licenses/netty-resolver-4.1.91.Final.jar.sha1 deleted file mode 100644 index bc57e2d0..00000000 --- a/licenses/netty-resolver-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04725d117d4b71ef0e743aa79062489b45472b26 \ No newline at end of file diff --git a/licenses/netty-resolver-4.1.93.Final.jar.sha1 b/licenses/netty-resolver-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..c795762c --- /dev/null +++ b/licenses/netty-resolver-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +3860e99075f9e078364ed38f6d6fc8825b7a168a \ No newline at end of file diff --git a/licenses/netty-transport-4.1.91.Final.jar.sha1 b/licenses/netty-transport-4.1.91.Final.jar.sha1 deleted file mode 100644 index 2562ece3..00000000 --- a/licenses/netty-transport-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2f6bd7143194ca842b535546a405c06aa993934 \ No newline at end of file diff --git a/licenses/netty-transport-4.1.93.Final.jar.sha1 b/licenses/netty-transport-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..687cade3 --- /dev/null +++ b/licenses/netty-transport-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +1a0894915c8027ce83b4d6a811c4e765955efd15 \ No newline at end of file diff --git a/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 deleted file mode 100644 index 6f45d642..00000000 --- a/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -80990b5885b8b67be096d7090cba18f05c67120e \ No newline at end of file diff --git a/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 new file mode 100644 index 00000000..0f0acb2e --- /dev/null +++ b/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 @@ -0,0 +1 @@ +f4fb8b4c8da539091f43abcbb9f0389e48807eea \ No newline at end of file diff --git a/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 b/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 index 8a8d4c4f..371c7cb8 100644 --- a/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 +++ b/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 @@ -1 +1 @@ -b3d405690aa95a02fbabe1fb2b83c0525041ee93 \ No newline at end of file +45ca12eb7e3e852a3d78f95015ce0310e6e40b93 \ No newline at end of file diff --git a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java index 00fa2f62..95b3348b 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java +++ b/src/main/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerPlugin.java @@ -53,6 +53,7 @@ import org.opensearch.performanceanalyzer.collectors.NodeDetailsCollector; import org.opensearch.performanceanalyzer.collectors.NodeStatsAllShardsMetricsCollector; import org.opensearch.performanceanalyzer.collectors.NodeStatsFixedShardsMetricsCollector; +import org.opensearch.performanceanalyzer.collectors.SearchBackPressureStatsCollector; import org.opensearch.performanceanalyzer.collectors.ShardIndexingPressureMetricsCollector; import org.opensearch.performanceanalyzer.collectors.ShardStateCollector; import org.opensearch.performanceanalyzer.collectors.ThreadPoolMetricsCollector; @@ -224,6 +225,10 @@ public PerformanceAnalyzerPlugin(final Settings settings, final java.nio.file.Pa scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new ClusterApplierServiceStatsCollector( performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( + new SearchBackPressureStatsCollector( + performanceAnalyzerController, configOverridesWrapper)); + scheduledMetricCollectorsExecutor.addScheduledMetricCollector( new AdmissionControlMetricsCollector()); scheduledMetricCollectorsExecutor.addScheduledMetricCollector( diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java new file mode 100644 index 00000000..372d0385 --- /dev/null +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java @@ -0,0 +1,665 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors; + +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatExceptionCode.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR; +import static org.opensearch.performanceanalyzer.commons.stats.metrics.StatMetrics.CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_EXECUTION_TIME; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.node.Node; +import org.opensearch.node.NodeService; +import org.opensearch.performanceanalyzer.commons.collectors.MetricStatus; +import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsProcessor; +import org.opensearch.performanceanalyzer.commons.metrics.PerformanceAnalyzerMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.opensearch.search.backpressure.SearchBackpressureService; +import org.opensearch.search.backpressure.stats.SearchShardTaskStats; +import org.opensearch.search.backpressure.stats.SearchTaskStats; + +public class SearchBackPressureStatsCollector extends PerformanceAnalyzerMetricsCollector + implements MetricsProcessor { + // SAMPLING TIME INTERVAL to collect search back pressure stats + public static final int SAMPLING_TIME_INTERVAL = + MetricsConfiguration.CONFIG_MAP.get(SearchBackPressureStatsCollector.class) + .samplingInterval; + private static final int KEYS_PATH_LENGTH = 0; + private static final Logger LOG = LogManager.getLogger(SearchBackPressureStatsCollector.class); + private static final ObjectMapper mapper; + + public static final String BOOTSTRAP_CLASS_NAME = "org.opensearch.bootstrap.Bootstrap"; + public static final String NODE_CLASS_NAME = "org.opensearch.node.Node"; + public static final String BOOTSTRAP_INSTANCE_FIELD_NAME = "INSTANCE"; + public static final String BOOTSTRAP_NODE_FIELD_NAME = "node"; + public static final String NODE_SERVICE_FIELD_NAME = "nodeService"; + + public static final String HEAP_USAGE_TRACKER_FIELD_NAME = "HEAP_USAGE_TRACKER"; + public static final String CPU_USAGE_TRACKER_FIELD_NAME = "CPU_USAGE_TRACKER"; + public static final String ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME = "ELAPSED_TIME_TRACKER"; + + // Headline for search back pressure metrics + public static final String PATH_TO_STORE_METRICS = "search_back_pressure"; + private String nodeId; + + // Metrics to be collected as a String and written in a JSON String + private final StringBuilder value; + private final PerformanceAnalyzerController controller; + private final ConfigOverridesWrapper configOverridesWrapper; + + static { + mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public SearchBackPressureStatsCollector( + PerformanceAnalyzerController controller, + ConfigOverridesWrapper configOverridesWrapper) { + super( + SAMPLING_TIME_INTERVAL, + SearchBackPressureStatsCollector.class.getSimpleName(), + CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_EXECUTION_TIME, + CLUSTER_APPLIER_SERVICE_STATS_COLLECTOR_ERROR); + + this.controller = controller; + this.configOverridesWrapper = configOverridesWrapper; + this.value = new StringBuilder(); + LOG.info("SearchBackPressureStatsCollector started"); + } + + private void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + private String getNodeId() { + return this.nodeId; + } + + @Override + public void collectMetrics(long startTime) { + SearchBackPressureStats currentSearchBackPressureStats = null; + try { + String jsonString = mapper.writeValueAsString(getSearchBackPressureStats()); + currentSearchBackPressureStats = + mapper.readValue(jsonString, SearchBackPressureStats.class); + + } catch (InvocationTargetException + | IllegalAccessException + | NoSuchMethodException + | NoSuchFieldException + | ClassNotFoundException + | JsonProcessingException ex) { + ex.printStackTrace(); + LOG.warn( + "No method found to get Search BackPressure Stats. " + + "Skipping SearchBackPressureStatsCollector. Error: " + + ex.getMessage()); + return; + } + + SearchBackPressureMetrics searchBackPressureMetrics = + new SearchBackPressureMetrics( + currentSearchBackPressureStats.getMode(), + getNodeId(), + currentSearchBackPressureStats.getSearchShardTaskStats(), + currentSearchBackPressureStats.getSearchTaskStats()); + + value.setLength(0); + // Append system current time and line seperator + value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) + .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + value.append(searchBackPressureMetrics.serialize()); + saveMetricValues(value.toString(), startTime); + } + + Field getField(String className, String fieldName) + throws NoSuchFieldException, ClassNotFoundException { + + Class BootStrapClass = Class.forName(className); + Field bootStrapField = BootStrapClass.getDeclaredField(fieldName); + + // set the field to be accessible + bootStrapField.setAccessible(true); + return bootStrapField; + } + + @VisibleForTesting + public Object getSearchBackPressureStats() + throws InvocationTargetException, IllegalAccessException, NoSuchMethodException, + NoSuchFieldException, NoSuchFieldError, ClassNotFoundException { + + // Get the static instance of Bootstrap + Object bootStrapSInstance = + getField(BOOTSTRAP_CLASS_NAME, BOOTSTRAP_INSTANCE_FIELD_NAME).get(null); + + // Get the Node instance from the Bootstrap instance + Node node = + (Node) + getField(BOOTSTRAP_CLASS_NAME, BOOTSTRAP_NODE_FIELD_NAME) + .get(bootStrapSInstance); + + NodeEnvironment nodeEnvironment = node.getNodeEnvironment(); + setNodeId(nodeEnvironment.nodeId()); + + // Get the NodeService instance from the Node instance + NodeService nodeService = + (NodeService) getField(NODE_CLASS_NAME, NODE_SERVICE_FIELD_NAME).get(node); + + String GET_STATS_METHOD_NAME = "nodeStats"; + Method method = SearchBackpressureService.class.getMethod(GET_STATS_METHOD_NAME); + + return method.invoke(nodeService.getSearchBackpressureService()); + } + + @Override + public String getMetricsPath(long startTime, String... keysPath) { + if (keysPath.length != KEYS_PATH_LENGTH) { + throw new RuntimeException("keys length should be " + KEYS_PATH_LENGTH); + } + return PerformanceAnalyzerMetrics.generatePath(startTime, PATH_TO_STORE_METRICS); + } + + public static class SearchBackPressureStats { + private SearchShardTaskStats searchShardTaskStats; + private String mode; + private SearchTaskStats searchTaskStats; + + @VisibleForTesting + @JsonCreator + public SearchBackPressureStats( + @JsonProperty("searchShardTaskStats") SearchShardTaskStats searchShardTaskStats, + @JsonProperty("mode") String mode, + @JsonProperty("searchTaskStats") SearchTaskStats searchTaskStats) { + this.searchShardTaskStats = searchShardTaskStats; + this.mode = mode; + this.searchTaskStats = searchTaskStats; + } + + public SearchBackPressureStats() {} + + // Getters and setters + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public SearchShardTaskStats getSearchShardTaskStats() { + return searchShardTaskStats; + } + + public void setSearchShardTaskStats(SearchShardTaskStats searchShardTaskStats) { + this.searchShardTaskStats = searchShardTaskStats; + } + + public SearchTaskStats getSearchTaskStats() { + return searchTaskStats; + } + + public void setSearchTaskStats(SearchTaskStats searchTaskStats) { + this.searchTaskStats = searchTaskStats; + } + } + + public static class SearchShardTaskStats { + private long cancellationCount; + private long limitReachedCount; + private Map resourceUsageTrackerStats; + + @JsonCreator + public SearchShardTaskStats( + @JsonProperty("cancellationCount") long cancellationCount, + @JsonProperty("limitReachedCount") long limitReachedCount, + @JsonProperty("resourceUsageTrackerStats") + Map resourceUsageTrackerStats) { + this.cancellationCount = cancellationCount; + this.limitReachedCount = limitReachedCount; + this.resourceUsageTrackerStats = resourceUsageTrackerStats; + } + + // Getters and Setters + public long getCancellationCount() { + return cancellationCount; + } + + public void setCancellationCount(long cancellationCount) { + this.cancellationCount = cancellationCount; + } + + public long getLimitReachedCount() { + return limitReachedCount; + } + + public void setLimitReachedCount(long limitReachedCount) { + this.limitReachedCount = limitReachedCount; + } + + public Map getResourceUsageTrackerStats() { + return resourceUsageTrackerStats; + } + + public void setResourceUsageTrackerStats( + Map resourceUsageTrackerStats) { + this.resourceUsageTrackerStats = resourceUsageTrackerStats; + } + } + + public static class SearchTaskStats { + private long cancellationCount; + private long limitReachedCount; + private Map resourceUsageTrackerStats; + + @JsonCreator + public SearchTaskStats( + @JsonProperty("cancellationCount") long cancellationCount, + @JsonProperty("limitReachedCount") long limitReachedCount, + @JsonProperty("resourceUsageTrackerStats") + Map resourceUsageTrackerStats) { + this.cancellationCount = cancellationCount; + this.limitReachedCount = limitReachedCount; + this.resourceUsageTrackerStats = resourceUsageTrackerStats; + } + + // Getters and Setters + public long getCancellationCount() { + return cancellationCount; + } + + public void setCancellationCount(long cancellationCount) { + this.cancellationCount = cancellationCount; + } + + public long getLimitReachedCount() { + return limitReachedCount; + } + + public void setLimitReachedCount(long limitReachedCount) { + this.limitReachedCount = limitReachedCount; + } + + public Map getResourceUsageTrackerStats() { + return resourceUsageTrackerStats; + } + + public void setResourceUsageTrackerStats( + Map resourceUsageTrackerStats) { + this.resourceUsageTrackerStats = resourceUsageTrackerStats; + } + } + + public static class ResourceUsageTrackerStats { + private long cancellationCount; + private long currentMax; + private long currentAvg; + private long rollingAvg; + private boolean fragment; + + @JsonCreator + public ResourceUsageTrackerStats( + @JsonProperty("cancellationCount") long cancellationCount, + @JsonProperty("currentMax") long currentMax, + @JsonProperty("currentAvg") long currentAvg, + @JsonProperty("rollingAvg") long rollingAvg, + @JsonProperty("fragment") boolean fragment) { + this.cancellationCount = cancellationCount; + this.currentMax = currentMax; + this.currentAvg = currentAvg; + this.rollingAvg = rollingAvg; + this.fragment = fragment; + } + // Getters and Setters + public long getCancellationCount() { + return cancellationCount; + } + + public void setCancellationCount(long cancellationCount) { + this.cancellationCount = cancellationCount; + } + + public long getCurrentMax() { + return currentMax; + } + + public void setCurrentMax(long currentMax) { + this.currentMax = currentMax; + } + + public long getCurrentAvg() { + return currentAvg; + } + + public void setCurrentAvg(long currentAvg) { + this.currentAvg = currentAvg; + } + + public long getRollingAvg() { + return rollingAvg; + } + + public void setRollingAvg(long rollingAvg) { + this.rollingAvg = rollingAvg; + } + + public boolean isFragment() { + return fragment; + } + + public void setFragment(boolean fragment) { + this.fragment = fragment; + } + } + + // SearchBackPressureMetrics() + /* + * SearchBackPressureMetrics( + * SearchTaskStats, + * mode, + * SearchShardTaskStats + * ) + */ + // Flatten the data fields for easier access + public static class SearchBackPressureMetrics extends MetricStatus { + private SearchShardTaskStats searchShardTaskStats; + private SearchTaskStats searchTaskStats; + + // private double searchBackPressureStatsTest; + private String mode; + private String nodeId; + + // SearchShardTaskStats related stats (General) + private long searchbp_shard_stats_cancellationCount; + private long searchbp_shard_stats_limitReachedCount; + + // SearchShardTaskStats related stats (resourceUsageTrackerStats) + // HEAP_USAGE_TRACKER + private long searchbp_shard_stats_resource_heap_usage_cancellationCount; + private long searchbp_shard_stats_resource_heap_usage_currentMax; + private long searchbp_shard_stats_resource_heap_usage_rollingAvg; + + // CPU_USAGE_TRACKER + private long searchbp_shard_stats_resource_cpu_usage_cancellationCount; + private long searchbp_shard_stats_resource_cpu_usage_currentMax; + private long searchbp_shard_stats_resource_cpu_usage_currentAvg; + + // ELAPSED_TIME_TRACKER + private long searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount; + private long searchbp_shard_stats_resource_elaspedtime_usage_currentMax; + private long searchbp_shard_stats_resource_elaspedtime_usage_currentAvg; + + // SearchTaskStats related stats (General) + private long searchbp_task_stats_cancellationCount; + private long searchbp_task_stats_limitReachedCount; + + // SearchTaskStats related stats (resourceUsageTrackerStats) + // HEAP_USAGE_TRACKER + private long searchbp_task_stats_resource_heap_usage_cancellationCount; + private long searchbp_task_stats_resource_heap_usage_currentMax; + private long searchbp_task_stats_resource_heap_usage_rollingAvg; + + // CPU_USAGE_TRACKER + private long searchbp_task_stats_resource_cpu_usage_cancellationCount; + private long searchbp_task_stats_resource_cpu_usage_currentMax; + private long searchbp_task_stats_resource_cpu_usage_currentAvg; + + // ELAPSED_TIME_TRACKER + private long searchbp_task_stats_resource_elaspedtime_usage_cancellationCount; + private long searchbp_task_stats_resource_elaspedtime_usage_currentMax; + private long searchbp_task_stats_resource_elaspedtime_usage_currentAvg; + + public SearchBackPressureMetrics( + String mode, + String nodeId, + SearchShardTaskStats searchShardTaskStats, + SearchTaskStats searchTaskStats) { + this.mode = mode; + this.nodeId = nodeId; + this.searchShardTaskStats = searchShardTaskStats; + this.searchTaskStats = searchTaskStats; + populate_shard_task_stats(searchShardTaskStats); + populate_task_stats(searchTaskStats); + } + + public void populate_shard_task_stats(SearchShardTaskStats searchShardTaskStats) { + this.searchbp_shard_stats_cancellationCount = + searchShardTaskStats.getCancellationCount(); + this.searchbp_shard_stats_limitReachedCount = + searchShardTaskStats.getLimitReachedCount(); + this.searchbp_shard_stats_resource_heap_usage_cancellationCount = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_shard_stats_resource_heap_usage_currentMax = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_shard_stats_resource_heap_usage_rollingAvg = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getRollingAvg(); + this.searchbp_shard_stats_resource_cpu_usage_cancellationCount = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_shard_stats_resource_cpu_usage_currentMax = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_shard_stats_resource_cpu_usage_currentAvg = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCurrentAvg(); + this.searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_shard_stats_resource_elaspedtime_usage_currentMax = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_shard_stats_resource_elaspedtime_usage_currentAvg = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCurrentAvg(); + } + + public void populate_task_stats(SearchTaskStats searchTaskStats) { + this.searchbp_task_stats_cancellationCount = searchTaskStats.getCancellationCount(); + this.searchbp_task_stats_limitReachedCount = searchTaskStats.getLimitReachedCount(); + this.searchbp_task_stats_resource_heap_usage_cancellationCount = + searchTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_task_stats_resource_heap_usage_currentMax = + searchTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_task_stats_resource_heap_usage_rollingAvg = + searchTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME) + .getRollingAvg(); + this.searchbp_task_stats_resource_cpu_usage_cancellationCount = + searchTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_task_stats_resource_cpu_usage_currentMax = + searchTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_task_stats_resource_cpu_usage_currentAvg = + searchTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME) + .getCurrentAvg(); + this.searchbp_task_stats_resource_elaspedtime_usage_cancellationCount = + searchTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCancellationCount(); + this.searchbp_task_stats_resource_elaspedtime_usage_currentMax = + searchTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCurrentMax(); + this.searchbp_task_stats_resource_elaspedtime_usage_currentAvg = + searchTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) + .getCurrentAvg(); + } + + @JsonProperty("searchbp_mode") + public String getSearchBackPressureStats_Mode() { + return this.mode; + } + + @JsonProperty("searchbp_nodeid") + public String getSearchBackPressureStats_NodeId() { + return this.nodeId; + } + + @JsonProperty("searchbp_shard_stats_cancellationCount") + public long getSearchbp_shard_stats_cancellationCount() { + return searchbp_shard_stats_cancellationCount; + } + + @JsonProperty("searchbp_shard_stats_limitReachedCount") + public long getSearchbp_shard_stats_limitReachedCount() { + return searchbp_shard_stats_limitReachedCount; + } + + @JsonProperty("searchbp_shard_stats_resource_heap_usage_cancellationCount") + public long getSearchbp_shard_stats_resource_heap_usage_cancellationCount() { + return searchbp_shard_stats_resource_heap_usage_cancellationCount; + } + + @JsonProperty("searchbp_shard_stats_resource_heap_usage_currentMax") + public long getSearchbp_shard_stats_resource_heap_usage_currentMax() { + return searchbp_shard_stats_resource_heap_usage_currentMax; + } + + @JsonProperty("searchbp_shard_stats_resource_heap_usage_rollingAvg") + public long getsearchbp_shard_stats_resource_heap_usage_rollingAvg() { + return searchbp_shard_stats_resource_heap_usage_rollingAvg; + } + + @JsonProperty("searchbp_shard_stats_resource_cpu_usage_cancellationCount") + public long getSearchbp_shard_stats_resource_cpu_usage_cancellationCount() { + return searchbp_shard_stats_resource_cpu_usage_cancellationCount; + } + + @JsonProperty("searchbp_shard_stats_resource_cpu_usage_currentMax") + public long getSearchbp_shard_stats_resource_cpu_usage_currentMax() { + return searchbp_shard_stats_resource_cpu_usage_currentMax; + } + + @JsonProperty("searchbp_shard_stats_resource_cpu_usage_currentAvg") + public long getSearchbp_shard_stats_resource_cpu_usage_currentAvg() { + return searchbp_shard_stats_resource_cpu_usage_currentAvg; + } + + @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount") + public long getSearchbp_shard_stats_resource_elaspedtime_usage_cancellationCount() { + return searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount; + } + + @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_currentMax") + public long getSearchbp_shard_stats_resource_elaspedtime_usage_currentMax() { + return searchbp_shard_stats_resource_elaspedtime_usage_currentMax; + } + + @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_currentAvg") + public long getSearchbp_shard_stats_resource_elaspedtime_usage_currentAvg() { + return searchbp_shard_stats_resource_elaspedtime_usage_currentAvg; + } + + @JsonProperty("searchbp_task_stats_cancellationCount") + public long getSearchbp_task_stats_cancellationCount() { + return searchbp_task_stats_cancellationCount; + } + + @JsonProperty("searchbp_task_stats_limitReachedCount") + public long getSearchbp_task_stats_limitReachedCount() { + return searchbp_task_stats_limitReachedCount; + } + + @JsonProperty("searchbp_task_stats_resource_heap_usage_cancellationCount") + public long getSearchbp_task_stats_resource_heap_usage_cancellationCount() { + return searchbp_task_stats_resource_heap_usage_cancellationCount; + } + + @JsonProperty("searchbp_task_stats_resource_heap_usage_currentMax") + public long getSearchbp_task_stats_resource_heap_usage_currentMax() { + return searchbp_task_stats_resource_heap_usage_currentMax; + } + + @JsonProperty("searchbp_task_stats_resource_heap_usage_rollingAvg") + public long getsearchbp_task_stats_resource_heap_usage_rollingAvg() { + return searchbp_task_stats_resource_heap_usage_rollingAvg; + } + + @JsonProperty("searchbp_task_stats_resource_cpu_usage_cancellationCount") + public long getSearchbp_task_stats_resource_cpu_usage_cancellationCount() { + return searchbp_task_stats_resource_cpu_usage_cancellationCount; + } + + @JsonProperty("searchbp_task_stats_resource_cpu_usage_currentMax") + public long getSearchbp_task_stats_resource_cpu_usage_currentMax() { + return searchbp_task_stats_resource_cpu_usage_currentMax; + } + + @JsonProperty("searchbp_task_stats_resource_cpu_usage_currentAvg") + public long getSearchbp_task_stats_resource_cpu_usage_currentAvg() { + return searchbp_task_stats_resource_cpu_usage_currentAvg; + } + + @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_cancellationCount") + public long getSearchbp_task_stats_resource_elaspedtime_usage_cancellationCount() { + return searchbp_task_stats_resource_elaspedtime_usage_cancellationCount; + } + + @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_currentMax") + public long getSearchbp_task_stats_resource_elaspedtime_usage_currentMax() { + return searchbp_task_stats_resource_elaspedtime_usage_currentMax; + } + + @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_currentAvg") + public long getSearchbp_task_stats_resource_elaspedtime_usage_currentAvg() { + return searchbp_task_stats_resource_elaspedtime_usage_currentAvg; + } + } +} diff --git a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java index ac705a98..8f20e4cc 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java +++ b/src/main/java/org/opensearch/performanceanalyzer/util/Utils.java @@ -44,6 +44,7 @@ public static void configureMetrics() { MetricsConfiguration.CONFIG_MAP.put( ClusterManagerThrottlingMetricsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ClusterApplierServiceStatsCollector.class, cdefault); + MetricsConfiguration.CONFIG_MAP.put(SearchBackPressureStatsCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ElectionTermCollector.class, cdefault); MetricsConfiguration.CONFIG_MAP.put(ShardIndexingPressureMetricsCollector.class, cdefault); } diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java new file mode 100644 index 00000000..be128f8c --- /dev/null +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java @@ -0,0 +1,169 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.performanceanalyzer.collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.event_process.Event; +import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; +import org.opensearch.performanceanalyzer.commons.metrics.PerformanceAnalyzerMetrics; +import org.opensearch.performanceanalyzer.config.PerformanceAnalyzerController; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +public class SearchBackPressureStatsCollectorTests { + private ObjectMapper mapper; + private long startTimeInMills; + private PerformanceAnalyzerController controller; + private ConfigOverridesWrapper configOverrides; + private SearchBackPressureStatsCollector searchBackPressureStatsCollector; + + // Required fields needed for search back pressure stats + private List required_fields_for_searchBackPressureStats = + Arrays.asList( + "searchbp_shard_stats_cancellationCount", + "searchbp_shard_stats_limitReachedCount", + "searchbp_shard_stats_resource_heap_usage_cancellationCount", + "searchbp_shard_stats_resource_heap_usage_currentMax", + "searchbp_shard_stats_resource_heap_usage_rollingAvg", + "searchbp_shard_stats_resource_cpu_usage_cancellationCount", + "searchbp_shard_stats_resource_cpu_usage_currentMax", + "searchbp_shard_stats_resource_cpu_usage_currentAvg", + "searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount", + "searchbp_shard_stats_resource_elaspedtime_usage_currentMax", + "searchbp_shard_stats_resource_elaspedtime_usage_currentAvg", + "searchbp_task_stats_cancellationCount", + "searchbp_task_stats_limitReachedCount", + "searchbp_task_stats_resource_heap_usage_cancellationCount", + "searchbp_task_stats_resource_heap_usage_currentMax", + "searchbp_task_stats_resource_heap_usage_rollingAvg", + "searchbp_task_stats_resource_cpu_usage_cancellationCount", + "searchbp_task_stats_resource_cpu_usage_currentMax", + "searchbp_task_stats_resource_cpu_usage_currentAvg", + "searchbp_task_stats_resource_elaspedtime_usage_cancellationCount", + "searchbp_task_stats_resource_elaspedtime_usage_currentMax", + "searchbp_task_stats_resource_elaspedtime_usage_currentAvg", + "searchbp_mode", + "searchbp_nodeid"); + + SearchBackPressureStatsCollector.ResourceUsageTrackerStats HEAP_USAGE_TRACKER_MOCK_STATS; + SearchBackPressureStatsCollector.ResourceUsageTrackerStats CPU_USAGE_TRACKER_MOCK_STATS; + SearchBackPressureStatsCollector.ResourceUsageTrackerStats ELAPSED_TIME_TRACKER_MOCK_STATS; + + /* + * Set the LOG property to be false + * Set the controller to be a mocked PerforamcneAnalyzerController.class + * Set the configWrapper to be a mocked ConfigOverridesWrapper + * Set the Collector to be a SearchBackPressureServiceCollector + * Set the ObjectMapper to be a new ObjectMapper() instance + */ + @Before + public void init() { + mapper = new ObjectMapper(); + MetricsConfiguration.CONFIG_MAP.put( + SearchBackPressureStatsCollector.class, MetricsConfiguration.cdefault); + System.setProperty("performanceanalyzer.metrics.log.enabled", "False"); + startTimeInMills = 1153721339; + controller = Mockito.mock(PerformanceAnalyzerController.class); + configOverrides = Mockito.mock(ConfigOverridesWrapper.class); + searchBackPressureStatsCollector = + new SearchBackPressureStatsCollector(controller, configOverrides); + + HEAP_USAGE_TRACKER_MOCK_STATS = + new SearchBackPressureStatsCollector.ResourceUsageTrackerStats(0, 0, 0, 0, false); + CPU_USAGE_TRACKER_MOCK_STATS = + new SearchBackPressureStatsCollector.ResourceUsageTrackerStats(0, 0, 0, 0, false); + ELAPSED_TIME_TRACKER_MOCK_STATS = + new SearchBackPressureStatsCollector.ResourceUsageTrackerStats(0, 0, 0, 0, false); + } + + @Test + public void testSearchBackPressureStats_saveMetricValues() { + Mockito.when( + controller.isCollectorEnabled( + configOverrides, "SearchBackPressureStatsCollector")) + .thenReturn(true); + searchBackPressureStatsCollector.saveMetricValues("search_back_pressure", startTimeInMills); + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + + assertEquals(1, metrics.size()); + assertEquals("search_back_pressure", metrics.get(0).value); + + try { + searchBackPressureStatsCollector.saveMetricValues( + "search_back_pressure", startTimeInMills, "dummy"); + assertTrue("Negative scenario test: Should have been a RuntimeException", true); + } catch (RuntimeException ex) { + // - expecting exception...1 values passed; 0 expected + // since keyPath does not match + } + } + + /* + * testSearchBackPressureStats_collectMetrics() test collectoMetrics() + * Mock the behavior getSearchBackPressureStats() to return a mock SearchBackPressureStats Instance + */ + @Test + public void testSearchBackPressureStats_collectMetrics() + throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, + JsonProcessingException, NoSuchFieldException, ClassNotFoundException { + String SEARCH_BACK_PRESSURE_MODE_FIELD_NAME = "searchbp_mode"; + SearchBackPressureStatsCollector spyCollector = + Mockito.spy(searchBackPressureStatsCollector); + + Map + resource_usage_mock_stats = + Map.ofEntries( + Map.entry("HEAP_USAGE_TRACKER", HEAP_USAGE_TRACKER_MOCK_STATS), + Map.entry("CPU_USAGE_TRACKER", CPU_USAGE_TRACKER_MOCK_STATS), + Map.entry("ELAPSED_TIME_TRACKER", ELAPSED_TIME_TRACKER_MOCK_STATS)); + + Mockito.doReturn( + new SearchBackPressureStatsCollector.SearchBackPressureStats( + new SearchBackPressureStatsCollector.SearchShardTaskStats( + 0, 0, resource_usage_mock_stats), + "MONITOR_ONLY", + new SearchBackPressureStatsCollector.SearchTaskStats( + 0, 0, resource_usage_mock_stats))) + .when(spyCollector) + .getSearchBackPressureStats(); + + Mockito.when( + controller.isCollectorEnabled( + configOverrides, + SearchBackPressureStatsCollector.class.getSimpleName())) + .thenReturn(true); + + spyCollector.collectMetrics(startTimeInMills); + List metrics = new ArrayList<>(); + PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + + assertEquals(1, metrics.size()); + + String[] lines = metrics.get(0).value.split(System.lineSeparator()); + Map map = mapper.readValue(lines[1], Map.class); + + // Verify requried fields are presented in the metrics + String jsonStr = lines[1]; + for (String required_field : required_fields_for_searchBackPressureStats) { + assertTrue(jsonStr.contains(required_field)); + } + } +} From d2225516408b6c5c59c4d5137c32487b397c586a Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Mon, 19 Jun 2023 15:49:04 -0700 Subject: [PATCH 2/6] Replace String fields with Constants in Commons Package (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- .../SearchBackPressureStatsCollector.java | 153 +++++++++++++----- 1 file changed, 115 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java index 372d0385..4e4d27c5 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java @@ -28,6 +28,7 @@ import org.opensearch.performanceanalyzer.commons.collectors.MetricStatus; import org.opensearch.performanceanalyzer.commons.collectors.PerformanceAnalyzerMetricsCollector; import org.opensearch.performanceanalyzer.commons.config.overrides.ConfigOverridesWrapper; +import org.opensearch.performanceanalyzer.commons.metrics.AllMetrics.SearchBackPressureStatsValue; import org.opensearch.performanceanalyzer.commons.metrics.MetricsConfiguration; import org.opensearch.performanceanalyzer.commons.metrics.MetricsProcessor; import org.opensearch.performanceanalyzer.commons.metrics.PerformanceAnalyzerMetrics; @@ -186,9 +187,18 @@ public static class SearchBackPressureStats { @VisibleForTesting @JsonCreator public SearchBackPressureStats( - @JsonProperty("searchShardTaskStats") SearchShardTaskStats searchShardTaskStats, - @JsonProperty("mode") String mode, - @JsonProperty("searchTaskStats") SearchTaskStats searchTaskStats) { + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_BACK_PRESSURE_STATS_SEARCH_SHARD_TASK_STATS) + SearchShardTaskStats searchShardTaskStats, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_BACK_PRESSURE_STATS_MODE) + String mode, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_BACK_PRESSURE_STATS_SEARCH_TASK_STATS) + SearchTaskStats searchTaskStats) { this.searchShardTaskStats = searchShardTaskStats; this.mode = mode; this.searchTaskStats = searchTaskStats; @@ -229,9 +239,17 @@ public static class SearchShardTaskStats { @JsonCreator public SearchShardTaskStats( - @JsonProperty("cancellationCount") long cancellationCount, - @JsonProperty("limitReachedCount") long limitReachedCount, - @JsonProperty("resourceUsageTrackerStats") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_SHARD_TASK_STATS_CANCELLATIONCOUNT) + long cancellationCount, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_SHARD_TASK_STATS_LIMITREACHEDCOUNT) + long limitReachedCount, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_SHARD_TASK_STATS_RESOURCE_USAGE_TRACKER_STATS) Map resourceUsageTrackerStats) { this.cancellationCount = cancellationCount; this.limitReachedCount = limitReachedCount; @@ -272,9 +290,17 @@ public static class SearchTaskStats { @JsonCreator public SearchTaskStats( - @JsonProperty("cancellationCount") long cancellationCount, - @JsonProperty("limitReachedCount") long limitReachedCount, - @JsonProperty("resourceUsageTrackerStats") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_TASK_STATS_CANCELLATIONCOUNT) + long cancellationCount, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_TASK_STATS_LIMITREACHEDCOUNT) + long limitReachedCount, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SEARCH_TASK_STATS_RESOURCE_USAGE_TRACKER_STATS) Map resourceUsageTrackerStats) { this.cancellationCount = cancellationCount; this.limitReachedCount = limitReachedCount; @@ -317,11 +343,26 @@ public static class ResourceUsageTrackerStats { @JsonCreator public ResourceUsageTrackerStats( - @JsonProperty("cancellationCount") long cancellationCount, - @JsonProperty("currentMax") long currentMax, - @JsonProperty("currentAvg") long currentAvg, - @JsonProperty("rollingAvg") long rollingAvg, - @JsonProperty("fragment") boolean fragment) { + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_RESOURCE_USAGE_TRACKER_STATS_CANCELLATIONCOUNT) + long cancellationCount, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_RESOURCE_USAGE_TRACKER_STATS_CURRENTMAX) + long currentMax, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_RESOURCE_USAGE_TRACKER_STATS_CURRENTAVG) + long currentAvg, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_RESOURCE_USAGE_TRACKER_STATS_ROLLINGAVG) + long rollingAvg, + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_RESOURCE_USAGE_TRACKER_STATS_FRAGMENT) + boolean fragment) { this.cancellationCount = cancellationCount; this.currentMax = currentMax; this.currentAvg = currentAvg; @@ -542,122 +583,158 @@ public void populate_task_stats(SearchTaskStats searchTaskStats) { .getCurrentAvg(); } - @JsonProperty("searchbp_mode") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_MODE) public String getSearchBackPressureStats_Mode() { return this.mode; } - @JsonProperty("searchbp_nodeid") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_NODEID) public String getSearchBackPressureStats_NodeId() { return this.nodeId; } - @JsonProperty("searchbp_shard_stats_cancellationCount") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_SHARD_STATS_CANCELLATIONCOUNT) public long getSearchbp_shard_stats_cancellationCount() { return searchbp_shard_stats_cancellationCount; } - @JsonProperty("searchbp_shard_stats_limitReachedCount") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_SHARD_STATS_LIMITREACHEDCOUNT) public long getSearchbp_shard_stats_limitReachedCount() { return searchbp_shard_stats_limitReachedCount; } - @JsonProperty("searchbp_shard_stats_resource_heap_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_HEAP_USAGE_CANCELLATIONCOUNT) public long getSearchbp_shard_stats_resource_heap_usage_cancellationCount() { return searchbp_shard_stats_resource_heap_usage_cancellationCount; } - @JsonProperty("searchbp_shard_stats_resource_heap_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_HEAP_USAGE_CURRENTMAX) public long getSearchbp_shard_stats_resource_heap_usage_currentMax() { return searchbp_shard_stats_resource_heap_usage_currentMax; } - @JsonProperty("searchbp_shard_stats_resource_heap_usage_rollingAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_HEAP_USAGE_ROLLINGAVG) public long getsearchbp_shard_stats_resource_heap_usage_rollingAvg() { return searchbp_shard_stats_resource_heap_usage_rollingAvg; } - @JsonProperty("searchbp_shard_stats_resource_cpu_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_CPU_USAGE_CANCELLATIONCOUNT) public long getSearchbp_shard_stats_resource_cpu_usage_cancellationCount() { return searchbp_shard_stats_resource_cpu_usage_cancellationCount; } - @JsonProperty("searchbp_shard_stats_resource_cpu_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_CPU_USAGE_CURRENTMAX) public long getSearchbp_shard_stats_resource_cpu_usage_currentMax() { return searchbp_shard_stats_resource_cpu_usage_currentMax; } - @JsonProperty("searchbp_shard_stats_resource_cpu_usage_currentAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_CPU_USAGE_CURRENTAVG) public long getSearchbp_shard_stats_resource_cpu_usage_currentAvg() { return searchbp_shard_stats_resource_cpu_usage_currentAvg; } - @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_ELASPEDTIME_USAGE_CANCELLATIONCOUNT) public long getSearchbp_shard_stats_resource_elaspedtime_usage_cancellationCount() { return searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount; } - @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_ELASPEDTIME_USAGE_CURRENTMAX) public long getSearchbp_shard_stats_resource_elaspedtime_usage_currentMax() { return searchbp_shard_stats_resource_elaspedtime_usage_currentMax; } - @JsonProperty("searchbp_shard_stats_resource_elaspedtime_usage_currentAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_SHARD_STATS_RESOURCE_ELASPEDTIME_USAGE_CURRENTAVG) public long getSearchbp_shard_stats_resource_elaspedtime_usage_currentAvg() { return searchbp_shard_stats_resource_elaspedtime_usage_currentAvg; } - @JsonProperty("searchbp_task_stats_cancellationCount") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_TASK_STATS_CANCELLATIONCOUNT) public long getSearchbp_task_stats_cancellationCount() { return searchbp_task_stats_cancellationCount; } - @JsonProperty("searchbp_task_stats_limitReachedCount") + @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_TASK_STATS_LIMITREACHEDCOUNT) public long getSearchbp_task_stats_limitReachedCount() { return searchbp_task_stats_limitReachedCount; } - @JsonProperty("searchbp_task_stats_resource_heap_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_HEAP_USAGE_CANCELLATIONCOUNT) public long getSearchbp_task_stats_resource_heap_usage_cancellationCount() { return searchbp_task_stats_resource_heap_usage_cancellationCount; } - @JsonProperty("searchbp_task_stats_resource_heap_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_HEAP_USAGE_CURRENTMAX) public long getSearchbp_task_stats_resource_heap_usage_currentMax() { return searchbp_task_stats_resource_heap_usage_currentMax; } - @JsonProperty("searchbp_task_stats_resource_heap_usage_rollingAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_HEAP_USAGE_ROLLINGAVG) public long getsearchbp_task_stats_resource_heap_usage_rollingAvg() { return searchbp_task_stats_resource_heap_usage_rollingAvg; } - @JsonProperty("searchbp_task_stats_resource_cpu_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_CPU_USAGE_CANCELLATIONCOUNT) public long getSearchbp_task_stats_resource_cpu_usage_cancellationCount() { return searchbp_task_stats_resource_cpu_usage_cancellationCount; } - @JsonProperty("searchbp_task_stats_resource_cpu_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_CPU_USAGE_CURRENTMAX) public long getSearchbp_task_stats_resource_cpu_usage_currentMax() { return searchbp_task_stats_resource_cpu_usage_currentMax; } - @JsonProperty("searchbp_task_stats_resource_cpu_usage_currentAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_CPU_USAGE_CURRENTAVG) public long getSearchbp_task_stats_resource_cpu_usage_currentAvg() { return searchbp_task_stats_resource_cpu_usage_currentAvg; } - @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_cancellationCount") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_ELASPEDTIME_USAGE_CANCELLATIONCOUNT) public long getSearchbp_task_stats_resource_elaspedtime_usage_cancellationCount() { return searchbp_task_stats_resource_elaspedtime_usage_cancellationCount; } - @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_currentMax") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_ELASPEDTIME_USAGE_CURRENTMAX) public long getSearchbp_task_stats_resource_elaspedtime_usage_currentMax() { return searchbp_task_stats_resource_elaspedtime_usage_currentMax; } - @JsonProperty("searchbp_task_stats_resource_elaspedtime_usage_currentAvg") + @JsonProperty( + SearchBackPressureStatsValue.Constants + .SEARCHBP_TASK_STATS_RESOURCE_ELASPEDTIME_USAGE_CURRENTAVG) public long getSearchbp_task_stats_resource_elaspedtime_usage_currentAvg() { return searchbp_task_stats_resource_elaspedtime_usage_currentAvg; } From 1d17ee8cdaaa7c601902c30c9252f4fc3dcbeb13 Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Tue, 20 Jun 2023 21:57:45 -0700 Subject: [PATCH 3/6] Remove jar files changes (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- licenses/guava-31.1-jre.jar.sha1 | 1 + licenses/j2objc-annotations-1.3.jar.sha1 | 1 + licenses/jackson-annotations-2.15.1.jar.sha1 | 1 + licenses/jackson-databind-2.15.1.jar.sha1 | 1 + licenses/jackson-module-paranamer-2.15.1.jar.sha1 | 1 + licenses/netty-buffer-4.1.91.Final.jar.sha1 | 1 + licenses/netty-codec-4.1.91.Final.jar.sha1 | 1 + licenses/netty-codec-http-4.1.91.Final.jar.sha1 | 1 + licenses/netty-codec-http2-4.1.91.Final.jar.sha1 | 1 + licenses/netty-codec-socks-4.1.91.Final.jar.sha1 | 1 + licenses/netty-common-4.1.91.Final.jar.sha1 | 1 + licenses/netty-handler-4.1.91.Final.jar.sha1 | 1 + licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 | 1 + licenses/netty-resolver-4.1.91.Final.jar.sha1 | 1 + licenses/netty-transport-4.1.91.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.91.Final.jar.sha1 | 1 + licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 | 2 +- 17 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 licenses/guava-31.1-jre.jar.sha1 create mode 100644 licenses/j2objc-annotations-1.3.jar.sha1 create mode 100644 licenses/jackson-annotations-2.15.1.jar.sha1 create mode 100644 licenses/jackson-databind-2.15.1.jar.sha1 create mode 100644 licenses/jackson-module-paranamer-2.15.1.jar.sha1 create mode 100644 licenses/netty-buffer-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-http-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-http2-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-codec-socks-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-common-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-handler-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-resolver-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-transport-4.1.91.Final.jar.sha1 create mode 100644 licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 diff --git a/licenses/guava-31.1-jre.jar.sha1 b/licenses/guava-31.1-jre.jar.sha1 new file mode 100644 index 00000000..e57390eb --- /dev/null +++ b/licenses/guava-31.1-jre.jar.sha1 @@ -0,0 +1 @@ +60458f877d055d0c9114d9e1a2efb737b4bc282c \ No newline at end of file diff --git a/licenses/j2objc-annotations-1.3.jar.sha1 b/licenses/j2objc-annotations-1.3.jar.sha1 new file mode 100644 index 00000000..0332207a --- /dev/null +++ b/licenses/j2objc-annotations-1.3.jar.sha1 @@ -0,0 +1 @@ +ba035118bc8bac37d7eff77700720999acd9986d \ No newline at end of file diff --git a/licenses/jackson-annotations-2.15.1.jar.sha1 b/licenses/jackson-annotations-2.15.1.jar.sha1 new file mode 100644 index 00000000..a3ade4ff --- /dev/null +++ b/licenses/jackson-annotations-2.15.1.jar.sha1 @@ -0,0 +1 @@ +092a90d3739e970e03b5971839e4fe51f13c1fa3 \ No newline at end of file diff --git a/licenses/jackson-databind-2.15.1.jar.sha1 b/licenses/jackson-databind-2.15.1.jar.sha1 new file mode 100644 index 00000000..47405b86 --- /dev/null +++ b/licenses/jackson-databind-2.15.1.jar.sha1 @@ -0,0 +1 @@ +ac9ba74d208faf356e4719a49e59c6ea9237c01d \ No newline at end of file diff --git a/licenses/jackson-module-paranamer-2.15.1.jar.sha1 b/licenses/jackson-module-paranamer-2.15.1.jar.sha1 new file mode 100644 index 00000000..50c1d67d --- /dev/null +++ b/licenses/jackson-module-paranamer-2.15.1.jar.sha1 @@ -0,0 +1 @@ +1eee39f6a814b12d92a936cd198657ff26970215 \ No newline at end of file diff --git a/licenses/netty-buffer-4.1.91.Final.jar.sha1 b/licenses/netty-buffer-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..158024bc --- /dev/null +++ b/licenses/netty-buffer-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +d8f180291c3501e931968ca7e40ae0323c4eacee \ No newline at end of file diff --git a/licenses/netty-codec-4.1.91.Final.jar.sha1 b/licenses/netty-codec-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..764a03d3 --- /dev/null +++ b/licenses/netty-codec-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +3044b8e325e33f72c96ac1ea51dda85bef090cc0 \ No newline at end of file diff --git a/licenses/netty-codec-http-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..ca956129 --- /dev/null +++ b/licenses/netty-codec-http-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +4519d2ff470941f0086214b19c9acf992868112f \ No newline at end of file diff --git a/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..d57336af --- /dev/null +++ b/licenses/netty-codec-http2-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +4ee7027e1653c6ee3f843191e0d932f29e8e14e1 \ No newline at end of file diff --git a/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..10d7478c --- /dev/null +++ b/licenses/netty-codec-socks-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +8f0a52677da411a8ab762c426d723c7f54471504 \ No newline at end of file diff --git a/licenses/netty-common-4.1.91.Final.jar.sha1 b/licenses/netty-common-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..deaad405 --- /dev/null +++ b/licenses/netty-common-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +93e5056462a242718e7689d81180d125c79d7723 \ No newline at end of file diff --git a/licenses/netty-handler-4.1.91.Final.jar.sha1 b/licenses/netty-handler-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..3e121e5d --- /dev/null +++ b/licenses/netty-handler-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +444cf41e4fe28c47ffebba5e77b9458a12f938a1 \ No newline at end of file diff --git a/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..116ed58f --- /dev/null +++ b/licenses/netty-handler-proxy-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +2e3e57eae1a61e4e5f558e39619186fec6c424d3 \ No newline at end of file diff --git a/licenses/netty-resolver-4.1.91.Final.jar.sha1 b/licenses/netty-resolver-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..bc57e2d0 --- /dev/null +++ b/licenses/netty-resolver-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +04725d117d4b71ef0e743aa79062489b45472b26 \ No newline at end of file diff --git a/licenses/netty-transport-4.1.91.Final.jar.sha1 b/licenses/netty-transport-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..2562ece3 --- /dev/null +++ b/licenses/netty-transport-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +c2f6bd7143194ca842b535546a405c06aa993934 \ No newline at end of file diff --git a/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 new file mode 100644 index 00000000..6f45d642 --- /dev/null +++ b/licenses/netty-transport-native-unix-common-4.1.91.Final.jar.sha1 @@ -0,0 +1 @@ +80990b5885b8b67be096d7090cba18f05c67120e \ No newline at end of file diff --git a/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 b/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 index 371c7cb8..8a8d4c4f 100644 --- a/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 +++ b/licenses/performance-analyzer-commons-1.0.0-SNAPSHOT.jar.sha1 @@ -1 +1 @@ -45ca12eb7e3e852a3d78f95015ce0310e6e40b93 \ No newline at end of file +b3d405690aa95a02fbabe1fb2b83c0525041ee93 \ No newline at end of file From 5348a9f96ea80b3fa6dff963484cfc0e38613f00 Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Tue, 20 Jun 2023 22:05:39 -0700 Subject: [PATCH 4/6] Remove licenses/ jar files changes (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- licenses/guava-32.0.1-jre.jar.sha1 | 1 - licenses/j2objc-annotations-2.8.jar.sha1 | 1 - licenses/jackson-annotations-2.15.2.jar.sha1 | 1 - licenses/jackson-databind-2.15.2.jar.sha1 | 1 - licenses/jackson-module-paranamer-2.15.2.jar.sha1 | 1 - licenses/netty-buffer-4.1.93.Final.jar.sha1 | 1 - licenses/netty-codec-4.1.93.Final.jar.sha1 | 1 - licenses/netty-codec-http-4.1.93.Final.jar.sha1 | 1 - licenses/netty-codec-http2-4.1.93.Final.jar.sha1 | 1 - licenses/netty-codec-socks-4.1.93.Final.jar.sha1 | 1 - licenses/netty-common-4.1.93.Final.jar.sha1 | 1 - licenses/netty-handler-4.1.93.Final.jar.sha1 | 1 - licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 | 1 - licenses/netty-resolver-4.1.93.Final.jar.sha1 | 1 - licenses/netty-transport-4.1.93.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.93.Final.jar.sha1 | 1 - 16 files changed, 16 deletions(-) delete mode 100644 licenses/guava-32.0.1-jre.jar.sha1 delete mode 100644 licenses/j2objc-annotations-2.8.jar.sha1 delete mode 100644 licenses/jackson-annotations-2.15.2.jar.sha1 delete mode 100644 licenses/jackson-databind-2.15.2.jar.sha1 delete mode 100644 licenses/jackson-module-paranamer-2.15.2.jar.sha1 delete mode 100644 licenses/netty-buffer-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-http-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-http2-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-codec-socks-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-common-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-handler-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-resolver-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-transport-4.1.93.Final.jar.sha1 delete mode 100644 licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 diff --git a/licenses/guava-32.0.1-jre.jar.sha1 b/licenses/guava-32.0.1-jre.jar.sha1 deleted file mode 100644 index 80dc9e93..00000000 --- a/licenses/guava-32.0.1-jre.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6e5d51a72d142f2d40a57dfb897188b36a95b489 \ No newline at end of file diff --git a/licenses/j2objc-annotations-2.8.jar.sha1 b/licenses/j2objc-annotations-2.8.jar.sha1 deleted file mode 100644 index dd70876c..00000000 --- a/licenses/j2objc-annotations-2.8.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c85270e307e7b822f1086b93689124b89768e273 \ No newline at end of file diff --git a/licenses/jackson-annotations-2.15.2.jar.sha1 b/licenses/jackson-annotations-2.15.2.jar.sha1 deleted file mode 100644 index f63416dd..00000000 --- a/licenses/jackson-annotations-2.15.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4724a65ac8e8d156a24898d50fd5dbd3642870b8 \ No newline at end of file diff --git a/licenses/jackson-databind-2.15.2.jar.sha1 b/licenses/jackson-databind-2.15.2.jar.sha1 deleted file mode 100644 index f16d80af..00000000 --- a/licenses/jackson-databind-2.15.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9353b021f10c307c00328f52090de2bdb4b6ff9c \ No newline at end of file diff --git a/licenses/jackson-module-paranamer-2.15.2.jar.sha1 b/licenses/jackson-module-paranamer-2.15.2.jar.sha1 deleted file mode 100644 index 11724827..00000000 --- a/licenses/jackson-module-paranamer-2.15.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afa71e1b9e9eee5664481cab589487d0a7f83ff4 \ No newline at end of file diff --git a/licenses/netty-buffer-4.1.93.Final.jar.sha1 b/licenses/netty-buffer-4.1.93.Final.jar.sha1 deleted file mode 100644 index 5c5a17a9..00000000 --- a/licenses/netty-buffer-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87fac21f4ef95157866b07b48e3c707a2f13c581 \ No newline at end of file diff --git a/licenses/netty-codec-4.1.93.Final.jar.sha1 b/licenses/netty-codec-4.1.93.Final.jar.sha1 deleted file mode 100644 index 2b12a111..00000000 --- a/licenses/netty-codec-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -503badb458b6586632be8d1f81aa4e5ab99a80fc \ No newline at end of file diff --git a/licenses/netty-codec-http-4.1.93.Final.jar.sha1 b/licenses/netty-codec-http-4.1.93.Final.jar.sha1 deleted file mode 100644 index 6719e882..00000000 --- a/licenses/netty-codec-http-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -36acf0c94d03eb6ecef78a749a32cbb7dc0c57b4 \ No newline at end of file diff --git a/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 b/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 deleted file mode 100644 index 02423842..00000000 --- a/licenses/netty-codec-http2-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0f1625b43bde13ec057da0d2fe381ded2547a70e \ No newline at end of file diff --git a/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 b/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 deleted file mode 100644 index 3e3f699a..00000000 --- a/licenses/netty-codec-socks-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81ca78969afc60073e47c3b7b361cc3839392c73 \ No newline at end of file diff --git a/licenses/netty-common-4.1.93.Final.jar.sha1 b/licenses/netty-common-4.1.93.Final.jar.sha1 deleted file mode 100644 index 2324a54d..00000000 --- a/licenses/netty-common-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1cfc49b91b0d3ddb30c9f7d8467e5d02ae8babdf \ No newline at end of file diff --git a/licenses/netty-handler-4.1.93.Final.jar.sha1 b/licenses/netty-handler-4.1.93.Final.jar.sha1 deleted file mode 100644 index 54e5b220..00000000 --- a/licenses/netty-handler-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10f7ed9d8e1bfcba416074c70e5388be96116bfc \ No newline at end of file diff --git a/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 b/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 deleted file mode 100644 index 1ac94e65..00000000 --- a/licenses/netty-handler-proxy-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c165c1efe1b9c0cc22546a057b530611a088768b \ No newline at end of file diff --git a/licenses/netty-resolver-4.1.93.Final.jar.sha1 b/licenses/netty-resolver-4.1.93.Final.jar.sha1 deleted file mode 100644 index c795762c..00000000 --- a/licenses/netty-resolver-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3860e99075f9e078364ed38f6d6fc8825b7a168a \ No newline at end of file diff --git a/licenses/netty-transport-4.1.93.Final.jar.sha1 b/licenses/netty-transport-4.1.93.Final.jar.sha1 deleted file mode 100644 index 687cade3..00000000 --- a/licenses/netty-transport-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a0894915c8027ce83b4d6a811c4e765955efd15 \ No newline at end of file diff --git a/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 b/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 deleted file mode 100644 index 0f0acb2e..00000000 --- a/licenses/netty-transport-native-unix-common-4.1.93.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4fb8b4c8da539091f43abcbb9f0389e48807eea \ No newline at end of file From d451e22ebf14286085bc36ed11a7f660b3c0c026 Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Tue, 20 Jun 2023 22:37:51 -0700 Subject: [PATCH 5/6] Refactor populate_task_stats() and populate_shard_task_stats() (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- .../SearchBackPressureStatsCollector.java | 124 +++++++----------- 1 file changed, 49 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java index 4e4d27c5..7bec682d 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java @@ -423,8 +423,6 @@ public void setFragment(boolean fragment) { public static class SearchBackPressureMetrics extends MetricStatus { private SearchShardTaskStats searchShardTaskStats; private SearchTaskStats searchTaskStats; - - // private double searchBackPressureStatsTest; private String mode; private String nodeId; @@ -482,105 +480,81 @@ public SearchBackPressureMetrics( } public void populate_shard_task_stats(SearchShardTaskStats searchShardTaskStats) { + // Create shard HEAP/CPU/TIME Stats ResourceUsageTrackerStats for simplification + ResourceUsageTrackerStats shard_heap_stats = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(HEAP_USAGE_TRACKER_FIELD_NAME); + + ResourceUsageTrackerStats shard_cpu_stats = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(CPU_USAGE_TRACKER_FIELD_NAME); + + ResourceUsageTrackerStats shard_time_stats = + searchShardTaskStats + .getResourceUsageTrackerStats() + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME); + this.searchbp_shard_stats_cancellationCount = searchShardTaskStats.getCancellationCount(); this.searchbp_shard_stats_limitReachedCount = searchShardTaskStats.getLimitReachedCount(); this.searchbp_shard_stats_resource_heap_usage_cancellationCount = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); + shard_heap_stats.getCancellationCount(); this.searchbp_shard_stats_resource_heap_usage_currentMax = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); + shard_heap_stats.getCurrentMax(); this.searchbp_shard_stats_resource_heap_usage_rollingAvg = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getRollingAvg(); + shard_heap_stats.getRollingAvg(); this.searchbp_shard_stats_resource_cpu_usage_cancellationCount = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); + shard_cpu_stats.getCancellationCount(); this.searchbp_shard_stats_resource_cpu_usage_currentMax = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); + shard_cpu_stats.getCurrentMax(); this.searchbp_shard_stats_resource_cpu_usage_currentAvg = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCurrentAvg(); + shard_cpu_stats.getCurrentAvg(); this.searchbp_shard_stats_resource_elaspedtime_usage_cancellationCount = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); + shard_time_stats.getCancellationCount(); this.searchbp_shard_stats_resource_elaspedtime_usage_currentMax = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); + shard_time_stats.getCurrentMax(); this.searchbp_shard_stats_resource_elaspedtime_usage_currentAvg = - searchShardTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCurrentAvg(); + shard_time_stats.getCurrentAvg(); } public void populate_task_stats(SearchTaskStats searchTaskStats) { - this.searchbp_task_stats_cancellationCount = searchTaskStats.getCancellationCount(); - this.searchbp_task_stats_limitReachedCount = searchTaskStats.getLimitReachedCount(); - this.searchbp_task_stats_resource_heap_usage_cancellationCount = + // Create task HEAP/CPU/TIME Stats ResourceUsageTrackerStats for simplification + ResourceUsageTrackerStats task_heap_stats = searchTaskStats .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); - this.searchbp_task_stats_resource_heap_usage_currentMax = + .get(HEAP_USAGE_TRACKER_FIELD_NAME); + + ResourceUsageTrackerStats task_cpu_stats = searchTaskStats .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); - this.searchbp_task_stats_resource_heap_usage_rollingAvg = + .get(CPU_USAGE_TRACKER_FIELD_NAME); + + ResourceUsageTrackerStats task_time_stats = searchTaskStats .getResourceUsageTrackerStats() - .get(HEAP_USAGE_TRACKER_FIELD_NAME) - .getRollingAvg(); + .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME); + + this.searchbp_task_stats_cancellationCount = searchTaskStats.getCancellationCount(); + this.searchbp_task_stats_limitReachedCount = searchTaskStats.getLimitReachedCount(); + this.searchbp_task_stats_resource_heap_usage_cancellationCount = + task_heap_stats.getCancellationCount(); + this.searchbp_task_stats_resource_heap_usage_currentMax = + task_heap_stats.getCurrentMax(); + this.searchbp_task_stats_resource_heap_usage_rollingAvg = + task_heap_stats.getRollingAvg(); this.searchbp_task_stats_resource_cpu_usage_cancellationCount = - searchTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); - this.searchbp_task_stats_resource_cpu_usage_currentMax = - searchTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); - this.searchbp_task_stats_resource_cpu_usage_currentAvg = - searchTaskStats - .getResourceUsageTrackerStats() - .get(CPU_USAGE_TRACKER_FIELD_NAME) - .getCurrentAvg(); + task_cpu_stats.getCancellationCount(); + this.searchbp_task_stats_resource_cpu_usage_currentMax = task_cpu_stats.getCurrentMax(); + this.searchbp_task_stats_resource_cpu_usage_currentAvg = task_cpu_stats.getCurrentAvg(); this.searchbp_task_stats_resource_elaspedtime_usage_cancellationCount = - searchTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCancellationCount(); + task_time_stats.getCancellationCount(); this.searchbp_task_stats_resource_elaspedtime_usage_currentMax = - searchTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCurrentMax(); + task_time_stats.getCurrentMax(); this.searchbp_task_stats_resource_elaspedtime_usage_currentAvg = - searchTaskStats - .getResourceUsageTrackerStats() - .get(ELAPSED_TIME_USAGE_TRACKER_FIELD_NAME) - .getCurrentAvg(); + task_time_stats.getCurrentAvg(); } @JsonProperty(SearchBackPressureStatsValue.Constants.SEARCHBP_MODE) From 1a0b4ed2074ddf79c1ea38597c6715b58f97872b Mon Sep 17 00:00:00 2001 From: CoderJeffrey Date: Fri, 23 Jun 2023 13:21:23 -0700 Subject: [PATCH 6/6] Add more Javadoc for SearchBackPressureCollector and UT (Signed-off-by: Jeffrey Liu ujeffliu@amazon.com) Signed-off-by: CoderJeffrey --- .../SearchBackPressureStatsCollector.java | 31 ++++++++++++++----- ...SearchBackPressureStatsCollectorTests.java | 17 +++++++--- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java index 7bec682d..fadbf135 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java +++ b/src/main/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollector.java @@ -72,6 +72,10 @@ public class SearchBackPressureStatsCollector extends PerformanceAnalyzerMetrics mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } + /* + * SearchBackPressureStatsCollector collects SearchBackPressure Related Stats from org.opensearch.search.backpressure.SearchBackpressureService + * Example Stats include the cancellation count of search tasks on shard level or node level + */ public SearchBackPressureStatsCollector( PerformanceAnalyzerController controller, ConfigOverridesWrapper configOverridesWrapper) { @@ -87,6 +91,9 @@ public SearchBackPressureStatsCollector( LOG.info("SearchBackPressureStatsCollector started"); } + /* + * NodeId is an additional field to be added to the searchbackpressure stats returned from SearchBackpressureService + */ private void setNodeId(String nodeId) { this.nodeId = nodeId; } @@ -124,11 +131,17 @@ public void collectMetrics(long startTime) { currentSearchBackPressureStats.getSearchShardTaskStats(), currentSearchBackPressureStats.getSearchTaskStats()); + // clear previous buffered value value.setLength(0); + // Append system current time and line seperator value.append(PerformanceAnalyzerMetrics.getJsonCurrentMilliSeconds()) .append(PerformanceAnalyzerMetrics.sMetricNewLineDelimitor); + + // Append search back pressure metrics value.append(searchBackPressureMetrics.serialize()); + + // Save metrics into /dev/shm folder saveMetricValues(value.toString(), startTime); } @@ -179,6 +192,9 @@ public String getMetricsPath(long startTime, String... keysPath) { return PerformanceAnalyzerMetrics.generatePath(startTime, PATH_TO_STORE_METRICS); } + /* + * POJO Class to deserialize the stats JSON String from SearchBackPressureService + */ public static class SearchBackPressureStats { private SearchShardTaskStats searchShardTaskStats; private String mode; @@ -232,6 +248,9 @@ public void setSearchTaskStats(SearchTaskStats searchTaskStats) { } } + /* + * POJO Class to deserialize Shard level Search Task Stats + */ public static class SearchShardTaskStats { private long cancellationCount; private long limitReachedCount; @@ -283,6 +302,9 @@ public void setResourceUsageTrackerStats( } } + /* + * POJO Class to deserialize Node level Search Task Stats + */ public static class SearchTaskStats { private long cancellationCount; private long limitReachedCount; @@ -411,15 +433,10 @@ public void setFragment(boolean fragment) { } } - // SearchBackPressureMetrics() /* - * SearchBackPressureMetrics( - * SearchTaskStats, - * mode, - * SearchShardTaskStats - * ) + * SearchBackPressureMetrics class to be stored + * Flatten the data fields for easier access */ - // Flatten the data fields for easier access public static class SearchBackPressureMetrics extends MetricStatus { private SearchShardTaskStats searchShardTaskStats; private SearchTaskStats searchTaskStats; diff --git a/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java b/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java index be128f8c..059a9302 100644 --- a/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java +++ b/src/test/java/org/opensearch/performanceanalyzer/collectors/SearchBackPressureStatsCollectorTests.java @@ -62,15 +62,17 @@ public class SearchBackPressureStatsCollectorTests { "searchbp_mode", "searchbp_nodeid"); + // Mock Instance for HEAP/CPU/ELAPSED_TIME usage SearchBackPressureStatsCollector.ResourceUsageTrackerStats HEAP_USAGE_TRACKER_MOCK_STATS; SearchBackPressureStatsCollector.ResourceUsageTrackerStats CPU_USAGE_TRACKER_MOCK_STATS; SearchBackPressureStatsCollector.ResourceUsageTrackerStats ELAPSED_TIME_TRACKER_MOCK_STATS; /* + * Required Config to be initialized * Set the LOG property to be false - * Set the controller to be a mocked PerforamcneAnalyzerController.class - * Set the configWrapper to be a mocked ConfigOverridesWrapper - * Set the Collector to be a SearchBackPressureServiceCollector + * Set the controller to be a Mock PerforamcneAnalyzerController.class + * Set the configWrapper to be a Mock ConfigOverridesWrapper + * Set the Collector to be a new SearchBackPressureServiceCollector * Set the ObjectMapper to be a new ObjectMapper() instance */ @Before @@ -93,6 +95,9 @@ public void init() { new SearchBackPressureStatsCollector.ResourceUsageTrackerStats(0, 0, 0, 0, false); } + /* + * testSearchBackPressureStats_collectMetrics() test saveMetricValues() for SearchBackPressureStatsCollector + */ @Test public void testSearchBackPressureStats_saveMetricValues() { Mockito.when( @@ -103,9 +108,11 @@ public void testSearchBackPressureStats_saveMetricValues() { List metrics = new ArrayList<>(); PerformanceAnalyzerMetrics.metricQueue.drainTo(metrics); + // Valid case testing assertEquals(1, metrics.size()); assertEquals("search_back_pressure", metrics.get(0).value); + // Exception case testing try { searchBackPressureStatsCollector.saveMetricValues( "search_back_pressure", startTimeInMills, "dummy"); @@ -117,7 +124,7 @@ public void testSearchBackPressureStats_saveMetricValues() { } /* - * testSearchBackPressureStats_collectMetrics() test collectoMetrics() + * testSearchBackPressureStats_collectMetrics() test collectoMetrics() for SearchBackPressureStatsCollector * Mock the behavior getSearchBackPressureStats() to return a mock SearchBackPressureStats Instance */ @Test @@ -160,7 +167,7 @@ public void testSearchBackPressureStats_collectMetrics() String[] lines = metrics.get(0).value.split(System.lineSeparator()); Map map = mapper.readValue(lines[1], Map.class); - // Verify requried fields are presented in the metrics + // Verify requried fields are all presented in the metrics String jsonStr = lines[1]; for (String required_field : required_fields_for_searchBackPressureStats) { assertTrue(jsonStr.contains(required_field));