From 48317a3a88e96a51fab693f5a790a7fd1f01eeae Mon Sep 17 00:00:00 2001 From: Josep Subirats Castell Date: Mon, 20 Dec 2021 10:38:03 +0100 Subject: [PATCH 01/96] Adds logback-classic-1.2 instrumentation --- .../logback-classic-1.2/build.gradle | 18 +++++ .../logbackclassic12/Constants.java | 16 +++++ .../Logger_Instrumentation.java | 18 +++++ .../Logger_InstrumentationTest.java | 68 +++++++++++++++++++ settings.gradle | 1 + 5 files changed, 121 insertions(+) create mode 100644 instrumentation/logback-classic-1.2/build.gradle create mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java create mode 100644 instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java diff --git a/instrumentation/logback-classic-1.2/build.gradle b/instrumentation/logback-classic-1.2/build.gradle new file mode 100644 index 0000000000..e736016dab --- /dev/null +++ b/instrumentation/logback-classic-1.2/build.gradle @@ -0,0 +1,18 @@ +jar { + manifest { attributes 'Implementation-Title': 'com.newrelic.instrumentation.logback-classic-1.2' } +} + +dependencies { + implementation(project(":agent-bridge")) + implementation("ch.qos.logback:logback-core:1.2.6") + implementation("ch.qos.logback:logback-classic:1.2.6") +} + +verifyInstrumentation { + passesOnly("ch.qos.logback:logback-core:[1.2.0,)") +} + +site { + title 'Logback' + type 'Framework' +} \ No newline at end of file diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java new file mode 100644 index 0000000000..7361933c68 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java @@ -0,0 +1,16 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +public enum Constants { + LOG_LINE("Logging/lines"), + LOG_LINE_LEVEL("Logging/lines/%s"); + + private String val; + + Constants(String val) { + this.val = val; + } + + public String getVal() { + return val; + } +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java new file mode 100644 index 0000000000..49d826463c --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java @@ -0,0 +1,18 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +import ch.qos.logback.classic.Level; +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import org.slf4j.Marker; + +@Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.Interface) +public abstract class Logger_Instrumentation { + private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, + final Throwable t) { + Weaver.callOriginal(); + NewRelic.incrementCounter(Constants.LOG_LINE.getVal()); + NewRelic.incrementCounter(String.format(Constants.LOG_LINE_LEVEL.getVal(), level)); + } +} diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java new file mode 100644 index 0000000000..289674bc0c --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java @@ -0,0 +1,68 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +import ch.qos.logback.classic.Level; +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.MetricsHelper; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import ch.qos.logback.classic.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.logbackclassic12" }) +public class Logger_InstrumentationTest { + + private static final String CAPTURED = "This log message should be captured"; + private static final String NOT_CAPTURED = "This message should NOT be captured"; + + @Test + public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { + // Given + final Logger logger = (Logger) LoggerFactory.getLogger(Logger_InstrumentationTest.class); + logger.setLevel(Level.INFO); + + // When + logger.trace(NOT_CAPTURED); + logger.debug(NOT_CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(8, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(3, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(4, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + @Test + public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToTrace() { + // Given + final Logger logger = (Logger) LoggerFactory.getLogger(Logger_InstrumentationTest.class); + logger.setLevel(Level.TRACE); + + // When + logger.trace(CAPTURED); + logger.debug(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(5, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index adf90343eb..fdf05cedc4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -225,6 +225,7 @@ include 'instrumentation:kafka-clients-heartbeat-2.1.0' include 'instrumentation:kafka-clients-metrics-0.10.0.0' include 'instrumentation:kafka-clients-metrics-2.0.0' include 'instrumentation:kafka-clients-spans-0.11.0.0' +include 'instrumentation:logback-classic-1.2' include 'instrumentation:mongodb-2.12' include 'instrumentation:mongodb-2.14' include 'instrumentation:mongodb-3.0' From 24efb84fb1559ae5bd278f7244a617df0a51eb94 Mon Sep 17 00:00:00 2001 From: Josep Subirats Castell Date: Mon, 20 Dec 2021 13:04:22 +0100 Subject: [PATCH 02/96] Addresses PR comments --- instrumentation/logback-classic-1.2/build.gradle | 1 - .../logbackclassic12/Constants.java | 16 ---------------- .../logbackclassic12/Logger_Instrumentation.java | 7 +++++-- .../Logger_InstrumentationTest.java | 2 +- 4 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java diff --git a/instrumentation/logback-classic-1.2/build.gradle b/instrumentation/logback-classic-1.2/build.gradle index e736016dab..b33a92dc51 100644 --- a/instrumentation/logback-classic-1.2/build.gradle +++ b/instrumentation/logback-classic-1.2/build.gradle @@ -4,7 +4,6 @@ jar { dependencies { implementation(project(":agent-bridge")) - implementation("ch.qos.logback:logback-core:1.2.6") implementation("ch.qos.logback:logback-classic:1.2.6") } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java deleted file mode 100644 index 7361933c68..0000000000 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Constants.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.nr.agent.instrumentation.logbackclassic12; - -public enum Constants { - LOG_LINE("Logging/lines"), - LOG_LINE_LEVEL("Logging/lines/%s"); - - private String val; - - Constants(String val) { - this.val = val; - } - - public String getVal() { - return val; - } -} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java index 49d826463c..308cd8e533 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java @@ -9,10 +9,13 @@ @Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.Interface) public abstract class Logger_Instrumentation { + private static final String LOG_LINES_METRIC = "Logging/lines"; + private static final String LOG_LINES_BY_LEVEL_METRIC_TEMPLATE = "Logging/lines/%s"; + private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { Weaver.callOriginal(); - NewRelic.incrementCounter(Constants.LOG_LINE.getVal()); - NewRelic.incrementCounter(String.format(Constants.LOG_LINE_LEVEL.getVal(), level)); + NewRelic.incrementCounter(LOG_LINES_METRIC); + NewRelic.incrementCounter(String.format(LOG_LINES_BY_LEVEL_METRIC_TEMPLATE, level)); } } diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java index 289674bc0c..c360397096 100644 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java @@ -1,13 +1,13 @@ package com.nr.agent.instrumentation.logbackclassic12; import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; import com.newrelic.agent.introspec.InstrumentationTestConfig; import com.newrelic.agent.introspec.InstrumentationTestRunner; import com.newrelic.agent.introspec.MetricsHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import ch.qos.logback.classic.Logger; import org.slf4j.LoggerFactory; @RunWith(InstrumentationTestRunner.class) From dc8b989036563ae87b0171be61182500eecbfe4a Mon Sep 17 00:00:00 2001 From: Josep Subirats Castell Date: Mon, 20 Dec 2021 14:02:50 +0100 Subject: [PATCH 03/96] Fixes instrumentation ranges, simplifies instrumentation for better performance --- instrumentation/logback-classic-1.2/build.gradle | 3 ++- .../logbackclassic12/Logger_Instrumentation.java | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/instrumentation/logback-classic-1.2/build.gradle b/instrumentation/logback-classic-1.2/build.gradle index b33a92dc51..2ae78febe0 100644 --- a/instrumentation/logback-classic-1.2/build.gradle +++ b/instrumentation/logback-classic-1.2/build.gradle @@ -8,7 +8,8 @@ dependencies { } verifyInstrumentation { - passesOnly("ch.qos.logback:logback-core:[1.2.0,)") + passes("ch.qos.logback:logback-classic:[1.0,)") + excludeRegex 'ch.qos.logback:logback-classic:1.3.*' } site { diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java index 308cd8e533..101ec46083 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java @@ -9,13 +9,11 @@ @Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.Interface) public abstract class Logger_Instrumentation { - private static final String LOG_LINES_METRIC = "Logging/lines"; - private static final String LOG_LINES_BY_LEVEL_METRIC_TEMPLATE = "Logging/lines/%s"; private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { Weaver.callOriginal(); - NewRelic.incrementCounter(LOG_LINES_METRIC); - NewRelic.incrementCounter(String.format(LOG_LINES_BY_LEVEL_METRIC_TEMPLATE, level)); + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter(String.format("Logging/lines/%s", level)); } } From 5c73287c981a1aab619a0a11de3976264ec863b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Tue, 18 Jan 2022 18:34:20 +0100 Subject: [PATCH 04/96] Run newrelic code before call original --- instrumentation/logback-classic-1.2/build.gradle | 5 +++-- .../logbackclassic12/Logger_Instrumentation.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/instrumentation/logback-classic-1.2/build.gradle b/instrumentation/logback-classic-1.2/build.gradle index 2ae78febe0..dcedf18f1c 100644 --- a/instrumentation/logback-classic-1.2/build.gradle +++ b/instrumentation/logback-classic-1.2/build.gradle @@ -8,8 +8,9 @@ dependencies { } verifyInstrumentation { - passes("ch.qos.logback:logback-classic:[1.0,)") - excludeRegex 'ch.qos.logback:logback-classic:1.3.*' + passesOnly("ch.qos.logback:logback-classic:[0.9.3,)") + excludeRegex '.*(alpha|groovyless).*' + excludeRegex 'ch.qos.logback:logback-classic:0.9.6' } site { diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java index 101ec46083..40bf17ee68 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java @@ -12,8 +12,8 @@ public abstract class Logger_Instrumentation { private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { - Weaver.callOriginal(); NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter(String.format("Logging/lines/%s", level)); + NewRelic.incrementCounter("Logging/lines" + level); + Weaver.callOriginal(); } } From 84700989d66bcc376f9d1ed7804298a91740962d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Wed, 19 Jan 2022 18:57:52 +0100 Subject: [PATCH 05/96] fix typo on metric name --- .../logbackclassic12/Logger_Instrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java index 40bf17ee68..0b85eca18f 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java @@ -13,7 +13,7 @@ public abstract class Logger_Instrumentation { private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines" + level); + NewRelic.incrementCounter("Logging/lines/" + level); Weaver.callOriginal(); } } From 3245d7f2e5afb2431d851d03590bb284884d157e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Fri, 21 Jan 2022 13:17:42 +0100 Subject: [PATCH 06/96] feat: add log4j-1 metrics instrumentation --- instrumentation/apache-log4j-1/build.gradle | 17 +++++ .../log4j1/Logger_Instrumentation.java | 17 +++++ .../log4j1/Logger_InstrumentationTest.java | 67 +++++++++++++++++++ settings.gradle | 1 + 4 files changed, 102 insertions(+) create mode 100644 instrumentation/apache-log4j-1/build.gradle create mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java create mode 100644 instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java diff --git a/instrumentation/apache-log4j-1/build.gradle b/instrumentation/apache-log4j-1/build.gradle new file mode 100644 index 0000000000..c9dadae145 --- /dev/null +++ b/instrumentation/apache-log4j-1/build.gradle @@ -0,0 +1,17 @@ +jar { + manifest { attributes 'Implementation-Title': 'com.newrelic.instrumentation.apache-log4j-1' } +} + +dependencies { + implementation(project(":agent-bridge")) + implementation("log4j:log4j:1.2.17") +} + +verifyInstrumentation { + passesOnly("log4j:log4j:[1.1.3,)") +} + +site { + title 'Log4j-1' + type 'Framework' +} diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java new file mode 100644 index 0000000000..a4d793af1a --- /dev/null +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java @@ -0,0 +1,17 @@ +package com.nr.agent.instrumentation.log4j1; + +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import org.apache.log4j.Priority; + +@Weave(originalName = "org.apache.log4j.Category") +public class Logger_Instrumentation { + + protected void forcedLog(String fqcn, Priority level, Object message, Throwable t) { + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + level.toString()); + Weaver.callOriginal(); + } + +} diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java new file mode 100644 index 0000000000..24a46c5c67 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java @@ -0,0 +1,67 @@ +package com.nr.agent.instrumentation.log4j1; + +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.MetricsHelper; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.log4j1" }) +public class Logger_InstrumentationTest { + + private static final String CAPTURED = "This log message should be captured"; + private static final String NOT_CAPTURED = "This message should NOT be captured"; + + @Test + public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { + // Given + final Logger logger = Logger.getLogger(Logger_InstrumentationTest.class); + logger.setLevel(Level.INFO); + + // When + logger.trace(NOT_CAPTURED); + logger.debug(NOT_CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(8, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(3, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(4, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + @Test + public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToTrace() { + // Given + final Logger logger = Logger.getLogger(Logger_InstrumentationTest.class); + logger.setLevel(Level.TRACE); + + // When + logger.trace(CAPTURED); + logger.debug(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(5, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } +} diff --git a/settings.gradle b/settings.gradle index 683a3eef7e..63776fa997 100644 --- a/settings.gradle +++ b/settings.gradle @@ -98,6 +98,7 @@ if (JavaVersion.current().isJava11Compatible()) { // Weaver Instrumentation include 'instrumentation:anorm-2.3' include 'instrumentation:anorm-2.4' +include 'instrumentation:apache-log4j-1' include 'instrumentation:aws-java-sdk-sqs-1.10.44' include 'instrumentation:aws-java-sdk-s3-1.2.13' include 'instrumentation:aws-java-sdk-s3-2.0' From ba228b77bbc7d8b3e4f606e837b66a30138ee90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Thu, 20 Jan 2022 16:47:39 +0100 Subject: [PATCH 07/96] feat: add log4j-2 metrics instrumentation --- instrumentation/apache-log4j-2/build.gradle | 18 +++++ .../log4j2/LoggerConfig_Instrumentation.java | 17 ++++ .../LoggerConfig_InstrumentationTest.java | 81 +++++++++++++++++++ settings.gradle | 1 + 4 files changed, 117 insertions(+) create mode 100644 instrumentation/apache-log4j-2/build.gradle create mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java create mode 100644 instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java diff --git a/instrumentation/apache-log4j-2/build.gradle b/instrumentation/apache-log4j-2/build.gradle new file mode 100644 index 0000000000..8257ce831b --- /dev/null +++ b/instrumentation/apache-log4j-2/build.gradle @@ -0,0 +1,18 @@ +jar { + manifest { attributes 'Implementation-Title': 'com.newrelic.instrumentation.apache-log4j-2' } +} + +dependencies { + implementation(project(":agent-bridge")) + implementation("org.apache.logging.log4j:log4j-core:2.17.1") +} + +verifyInstrumentation { + passesOnly("org.apache.logging.log4j:log4j-core:[2.0.0,)") + excludeRegex '.*(alpha|beta|rc).*' +} + +site { + title 'Log4j2' + type 'Framework' +} diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java new file mode 100644 index 0000000000..fae166c372 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java @@ -0,0 +1,17 @@ +package com.nr.agent.instrumentation.log4j2; + +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import org.apache.logging.log4j.core.LogEvent; + +@Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig") +public class LoggerConfig_Instrumentation { + + protected void callAppenders(LogEvent event) { + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); + Weaver.callOriginal(); + } + +} \ No newline at end of file diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java new file mode 100644 index 0000000000..ab0f0780cf --- /dev/null +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -0,0 +1,81 @@ +package com.nr.agent.instrumentation.log4j2; + +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.MetricsHelper; +import junit.framework.TestCase; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.log4j2" }) +public class LoggerConfig_InstrumentationTest extends TestCase { + + private static final String CAPTURED = "This log message should be captured"; + private static final String NOT_CAPTURED = "This message should NOT be captured"; + + @Test + public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { + // Given + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + setLoggerLevel(Level.INFO); + + // When + logger.trace(NOT_CAPTURED); + logger.debug(NOT_CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(8, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(3, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(4, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + @Test + public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToTrace() { + // Given + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + setLoggerLevel(Level.TRACE); + + // When + logger.trace(CAPTURED); + logger.debug(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(5, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + private void setLoggerLevel(Level level) { + final LoggerContext context = (LoggerContext) LogManager.getContext(false); + final Configuration configuration = context.getConfiguration(); + final LoggerConfig rootConfig = configuration.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + rootConfig.setLevel(level); + context.updateLoggers(); + } + +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 683a3eef7e..fcc3d303c7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -118,6 +118,7 @@ include 'instrumentation:akka-http-core-2.13_10.1.8' include 'instrumentation:akka-http-core-2.13_10.2.0' include 'instrumentation:akka-http-core-10.0' include 'instrumentation:akka-http-core-10.2.0' +include 'instrumentation:apache-log4j-2' include 'instrumentation:async-http-client-2.0.0' include 'instrumentation:async-http-client-2.1.0' include 'instrumentation:cassandra-datastax-2.1.2' From c0a3ff20c62a3971450a730df9419011245e24b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Mon, 24 Jan 2022 11:37:06 +0100 Subject: [PATCH 08/96] fix: rename class to match the one being Weaved --- ...r_Instrumentation.java => Category_Instrumentation.java} | 2 +- ...mentationTest.java => Category_InstrumentationTest.java} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/{Logger_Instrumentation.java => Category_Instrumentation.java} (92%) rename instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/{Logger_InstrumentationTest.java => Category_InstrumentationTest.java} (92%) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java similarity index 92% rename from instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java rename to instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java index a4d793af1a..4478c102ab 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Logger_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java @@ -6,7 +6,7 @@ import org.apache.log4j.Priority; @Weave(originalName = "org.apache.log4j.Category") -public class Logger_Instrumentation { +public class Category_Instrumentation { protected void forcedLog(String fqcn, Priority level, Object message, Throwable t) { NewRelic.incrementCounter("Logging/lines"); diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java similarity index 92% rename from instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java rename to instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java index 24a46c5c67..a9a973477a 100644 --- a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Logger_InstrumentationTest.java +++ b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java @@ -11,7 +11,7 @@ @RunWith(InstrumentationTestRunner.class) @InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.log4j1" }) -public class Logger_InstrumentationTest { +public class Category_InstrumentationTest { private static final String CAPTURED = "This log message should be captured"; private static final String NOT_CAPTURED = "This message should NOT be captured"; @@ -19,7 +19,7 @@ public class Logger_InstrumentationTest { @Test public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { // Given - final Logger logger = Logger.getLogger(Logger_InstrumentationTest.class); + final Logger logger = Logger.getLogger(Category_InstrumentationTest.class); logger.setLevel(Level.INFO); // When @@ -46,7 +46,7 @@ public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { @Test public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToTrace() { // Given - final Logger logger = Logger.getLogger(Logger_InstrumentationTest.class); + final Logger logger = Logger.getLogger(Category_InstrumentationTest.class); logger.setLevel(Level.TRACE); // When From ffdad92eb0787222311ba7e753a5d5ed24c2bbbd Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 24 Jan 2022 10:20:39 -0800 Subject: [PATCH 09/96] Intitial prototype sending LogEvents to log_event_data endpoint --- .../com/newrelic/agent/bridge/NoOpAgent.java | 5 + .../com/newrelic/agent/model/LogEvent.java | 46 ++ .../IntrospectorLogSenderService.java | 153 ++++++ .../internal/IntrospectorRPMService.java | 5 + .../internal/IntrospectorServiceManager.java | 8 + .../classic/Logger_Instrumentation.java | 36 ++ .../spi/LoggingEvent_Instrumentation.java | 202 ++++++++ .../logbackclassic12/AgentUtil.java | 22 + .../Logger_Instrumentation.java | 19 - .../Logger_InstrumentationTest.java | 2 +- .../java/com/newrelic/agent/AgentImpl.java | 5 + .../com/newrelic/agent/DummyTransaction.java | 12 + .../newrelic/agent/HarvestServiceImpl.java | 11 +- .../java/com/newrelic/agent/IRPMService.java | 3 + .../java/com/newrelic/agent/MetricNames.java | 2 + .../java/com/newrelic/agent/RPMService.java | 34 ++ .../java/com/newrelic/agent/Transaction.java | 14 + .../com/newrelic/agent/TransactionData.java | 4 + .../newrelic/agent/config/AgentConfig.java | 5 + .../agent/config/AgentConfigImpl.java | 15 +- .../agent/config/LogSenderConfig.java | 19 + .../agent/config/LogSenderConfigImpl.java | 59 +++ .../agent/service/ServiceManager.java | 3 + .../agent/service/ServiceManagerImpl.java | 11 + .../logging/LogSenderHarvestableImpl.java | 39 ++ .../service/logging/LogSenderService.java | 43 ++ .../service/logging/LogSenderServiceImpl.java | 486 ++++++++++++++++++ .../agent/transport/CollectorMethods.java | 1 + .../newrelic/agent/transport/DataSender.java | 6 + .../agent/transport/DataSenderImpl.java | 27 + .../com/newrelic/agent/BaseRPMService.java | 5 + .../com/newrelic/agent/MockDataSender.java | 5 + .../com/newrelic/agent/MockRPMService.java | 13 + .../newrelic/agent/MockServiceManager.java | 13 + .../java/com/newrelic/api/agent/Agent.java | 9 + .../com/newrelic/api/agent/NoOpAgent.java | 5 + 36 files changed, 1325 insertions(+), 22 deletions(-) create mode 100644 agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java create mode 100644 instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java delete mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java index 8b5d48b74f..4e53dff0fc 100644 --- a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java @@ -63,6 +63,11 @@ public Insights getInsights() { return NoOpInsights.INSTANCE; } + @Override + public Insights getLogSender() { + return NoOpInsights.INSTANCE; + } + @Override public boolean startAsyncActivity(Object activityContext) { return false; diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java new file mode 100644 index 0000000000..18a4ddcad3 --- /dev/null +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -0,0 +1,46 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.model; + +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONStreamAware; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; +import java.util.Map; + +public class LogEvent extends AnalyticsEvent implements JSONStreamAware { + + private volatile float mutablePriority; + + public LogEvent(String type, long timestamp, Map attributes, float priority) { + super(type, timestamp, priority, attributes); + this.mutablePriority = priority; + } + + public void setPriority(float priority) { + this.mutablePriority = priority; + } + + @Override + public float getPriority() { + return mutablePriority; + } + + @SuppressWarnings("unchecked") + @Override + public void writeJSONString(Writer out) throws IOException { + JSONObject intrinsics = new JSONObject(); + intrinsics.put("type", getType()); + intrinsics.put("timestamp", getTimestamp()); + JSONArray.writeJSONString(Arrays.asList(intrinsics, getMutableUserAttributes()), out); + } + +} diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java new file mode 100644 index 0000000000..39ff5a7da5 --- /dev/null +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -0,0 +1,153 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.introspec.internal; + +import com.newrelic.agent.Agent; +import com.newrelic.agent.config.AgentConfig; +import com.newrelic.agent.deps.com.google.common.collect.LinkedListMultimap; +import com.newrelic.agent.deps.com.google.common.collect.ListMultimap; +import com.newrelic.agent.deps.com.google.common.collect.Multimaps; +import com.newrelic.agent.deps.com.google.common.collect.Maps; +import com.newrelic.agent.introspec.Event; +import com.newrelic.agent.logging.IAgentLogger; +import com.newrelic.agent.model.AnalyticsEvent; +import com.newrelic.agent.model.LogEvent; +import com.newrelic.agent.service.logging.LogSenderService; +import com.newrelic.agent.tracing.DistributedTraceServiceImpl; +import com.newrelic.api.agent.Insights; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class IntrospectorLogSenderService implements LogSenderService { + + private static String SERVICE_NAME = "LogSenderService"; + private ListMultimap events = Multimaps.synchronizedListMultimap(LinkedListMultimap.create()); + + @Override + public String getName() { + return SERVICE_NAME; + } + + @Override + public void start() throws Exception { + } + + @Override + public void stop() throws Exception { + } + + @Override + public boolean isEnabled() { + return true; + } + + @Override + public IAgentLogger getLogger() { + return Agent.LOG; + } + + @Override + public boolean isStarted() { + return true; + } + + @Override + public boolean isStopped() { + return false; + } + + @Override + public boolean isStartedOrStarting() { + return true; + } + + @Override + public boolean isStoppedOrStopping() { + return false; + } + + @Override + public void recordCustomEvent(String eventType, Map attributes) { + if (AnalyticsEvent.isValidType(eventType)) { + Map atts = Maps.newHashMap(attributes); + LogEvent event = new LogEvent(eventType, System.currentTimeMillis(), atts, DistributedTraceServiceImpl.nextTruncatedFloat()); + storeEvent("TestApp", event); + } + } + + @Override + public Insights getTransactionInsights(AgentConfig config) { + return this; + } + + @Override + public void storeEvent(String appName, LogEvent event) { + events.put(event.getType(), event); + } + + @Override + public void addHarvestableToService(String s) { + } + + public Collection getEventTypes() { + return Collections.unmodifiableCollection(events.keys()); + } + + public Collection getEvents(String type) { + List currentEvents = events.get(type); + if (currentEvents == null) { + return null; + } + List output = new ArrayList<>(currentEvents.size()); + for (LogEvent current : currentEvents) { + output.add(new EventImpl(current.getType(), current.getUserAttributesCopy())); + } + return output; + } + + public void clear() { + events.clear(); + } + + @Override + public void harvestEvents(String appName) { + } + + @Override + public String getEventHarvestIntervalMetric() { + return ""; + } + + @Override + public String getReportPeriodInSecondsMetric() { + return ""; + } + + @Override + public String getEventHarvestLimitMetric() { + return ""; + } + + @Override + public int getMaxSamplesStored() { + return 0; + } + + @Override + public void setMaxSamplesStored(int maxSamplesStored) { + } + + @Override + public void clearReservoir() { + events.clear(); + } +} diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java index 3fbacdada5..7ac384393f 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java @@ -12,6 +12,7 @@ import com.newrelic.agent.errors.TracedError; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.service.AbstractService; @@ -149,6 +150,10 @@ public void sendAnalyticsEvents(int reservoirSize, int eventsSeen, Collection events) { } + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) { + } + @Override public void sendErrorEvents(int reservoirSize, int eventsSeen, Collection events) { } diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorServiceManager.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorServiceManager.java index e9fc9b427e..3fb9b4d427 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorServiceManager.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorServiceManager.java @@ -40,6 +40,7 @@ import com.newrelic.agent.service.ServiceManager; import com.newrelic.agent.service.analytics.*; import com.newrelic.agent.service.async.AsyncTransactionService; +import com.newrelic.agent.service.logging.LogSenderService; import com.newrelic.agent.service.module.JarCollectorService; import com.newrelic.agent.sql.SqlTraceService; import com.newrelic.agent.sql.SqlTraceServiceImpl; @@ -80,6 +81,7 @@ class IntrospectorServiceManager extends AbstractService implements ServiceManag private volatile AsyncTransactionService asyncTxService; private volatile CircuitBreakerService circuitBreakerService; private volatile InsightsService insightsService; + private volatile LogSenderService logSenderService; private volatile DistributedTraceServiceImpl distributedTraceService; private volatile SpanEventsService spanEventsService; private volatile SourceLanguageService sourceLanguageService; @@ -141,6 +143,7 @@ private void setup(Map config) { harvestService = new IntrospectorHarvestService(); sqlTraceService = new SqlTraceServiceImpl(); insightsService = new IntrospectorInsightsService(); + logSenderService = new IntrospectorLogSenderService(); expirationService = new ExpirationService(); dbService = new DatabaseService(); jarCollectorService = new IgnoringJarCollectorService(); @@ -389,6 +392,11 @@ public InsightsService getInsights() { return insightsService; } + @Override + public LogSenderService getLogSenderService() { + return logSenderService; + } + @Override public CircuitBreakerService getCircuitBreakerService() { return circuitBreakerService; diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java new file mode 100644 index 0000000000..8936088678 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -0,0 +1,36 @@ +package ch.qos.logback.classic; + +import ch.qos.logback.classic.Level; +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import org.slf4j.Marker; + +@Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.ExactClass) +public abstract class Logger_Instrumentation { + + private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, + final Throwable t) { + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + level); + +// msg = msg + getLinkingMetadataAsString(); + +// LoggingEvent loggingEvent = new LoggingEvent(); +// loggingEvent. + +// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + + // TODO conditional check if logs should be decorated and sent to NR, send relevant event if so + // check if only in transaction??? + // check log level, only send up certain log levels by default? + Weaver.callOriginal(); + } + + // TODO look into weaving ch.qos.logback.classic.AsyncAppender, ch.qos.logback.core.encoder.Encoder, and ch.qos.logback.core.LayoutBase. + // AsyncAppender could be a spot to capture New Relic trace data + // Encoder could be a spot to convert the logs into NR JSON + // LayoutBase could be a spot to format the logs + // Look into org.slf4j.Logger as a possible common interface weave point? probably doesn't make sense +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java new file mode 100644 index 0000000000..19e88fb644 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -0,0 +1,202 @@ +package ch.qos.logback.classic.spi; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; +import org.slf4j.Marker; + +import java.util.HashMap; +import java.util.Map; + +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getLinkingMetadataAsMap; + +@Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) +public class LoggingEvent_Instrumentation implements ILoggingEvent { + /** + * Fully qualified name of the calling Logger class. This field does not + * survive serialization. + *

+ *

+ * Note that the getCallerInformation() method relies on this fact. + */ + transient String fqnOfLoggerClass; + + /** + * The name of thread in which this logging event was generated. + */ + private String threadName; + + private String loggerName; + private LoggerContext loggerContext; + private LoggerContextVO loggerContextVO; + + /** + * Level of logging event. + *

+ *

+ * This field should not be accessed directly. You should use the + * {@link #getLevel} method instead. + *

+ */ + private transient Level level; + + private String message; + + // we gain significant space at serialization time by marking + // formattedMessage as transient and constructing it lazily in + // getFormattedMessage() + transient String formattedMessage; + + private transient Object[] argumentArray; + + private ThrowableProxy throwableProxy; + + private StackTraceElement[] callerDataArray; + + private Marker marker; + + private Map mdcPropertyMap; + + /** + * The number of milliseconds elapsed from 1/1/1970 until logging event was + * created. + */ + private long timeStamp; + + public LoggingEvent_Instrumentation() { + } + + public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { +// "message": "...", +// "timestamp": 1641579045527, +// "thread.name": "http-nio-8080-exec-7", +// "log.level": "ERROR", +// "logger.name": "org.springframework.samples.petclinic.system.CrashController", +// "class.name": "org.springframework.samples.petclinic.system.CrashController", +// "method.name": "triggerException", +// "line.number": 41, + + HashMap logEventMap = new HashMap<>(getLinkingMetadataAsMap()); + logEventMap.put("message", message); + logEventMap.put("timeStamp", timeStamp); + logEventMap.put("log.level", level); + logEventMap.put("logger.name", logger.getName()); + logEventMap.put("class.name", fqcn); + + this.fqnOfLoggerClass = fqcn; + this.loggerName = logger.getName(); + this.loggerContext = logger.getLoggerContext(); + this.loggerContextVO = loggerContext.getLoggerContextRemoteView(); + this.level = level; + + // Append New Relic linking metadata from agent to log message + // TODO conditional checks based on config + // Should log be decorated? Should the decoration persist in the log file/console? Only log at certain log level? + this.message = message + " NR-LINKING-METADATA: " + AgentUtil.getLinkingMetadataAsString(); + this.argumentArray = argArray; + + if (throwable == null) { + throwable = extractThrowableAnRearrangeArguments(argArray); + } + + if (throwable != null) { + logEventMap.put("throwable", throwable); + this.throwableProxy = new ThrowableProxy(throwable); + LoggerContext lc = logger.getLoggerContext(); + if (lc.isPackagingDataEnabled()) { + this.throwableProxy.calculatePackagingData(); + } + } + + timeStamp = System.currentTimeMillis(); + + NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); + } + + private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { + return Weaver.callOriginal(); + } + + @Override + public String getThreadName() { + return Weaver.callOriginal(); + } + + @Override + public Level getLevel() { + return Weaver.callOriginal(); + } + + @Override + public String getMessage() { + return Weaver.callOriginal(); + } + + @Override + public Object[] getArgumentArray() { + return Weaver.callOriginal(); + } + + @Override + public String getFormattedMessage() { + return Weaver.callOriginal(); + } + + @Override + public String getLoggerName() { + return Weaver.callOriginal(); + } + + @Override + public LoggerContextVO getLoggerContextVO() { + return Weaver.callOriginal(); + } + + @Override + public IThrowableProxy getThrowableProxy() { + return Weaver.callOriginal(); + } + + @Override + public StackTraceElement[] getCallerData() { + return Weaver.callOriginal(); + } + + @Override + public boolean hasCallerData() { + return Weaver.callOriginal(); + } + + @Override + public Marker getMarker() { + return Weaver.callOriginal(); + } + + @Override + public Map getMDCPropertyMap() { + return Weaver.callOriginal(); + } + + /** + * @deprecated + */ + @Override + public Map getMdc() { + return Weaver.callOriginal(); + } + + @Override + public long getTimeStamp() { + return Weaver.callOriginal(); + } + + @Override + public void prepareForDeferredProcessing() { + Weaver.callOriginal(); + } +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java new file mode 100644 index 0000000000..2f77f4bc78 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -0,0 +1,22 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +import com.newrelic.api.agent.NewRelic; + +import java.util.Map; + +public class AgentUtil { + + public static Map getLinkingMetadataAsMap() { + Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + // TODO omit attributes that don't have values + // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} + return linkingMetadata; + } + + public static String getLinkingMetadataAsString() { + Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + // TODO omit attributes that don't have values + // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} + return linkingMetadata.toString(); + } +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java deleted file mode 100644 index 0b85eca18f..0000000000 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/Logger_Instrumentation.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.nr.agent.instrumentation.logbackclassic12; - -import ch.qos.logback.classic.Level; -import com.newrelic.api.agent.NewRelic; -import com.newrelic.api.agent.weaver.MatchType; -import com.newrelic.api.agent.weaver.Weave; -import com.newrelic.api.agent.weaver.Weaver; -import org.slf4j.Marker; - -@Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.Interface) -public abstract class Logger_Instrumentation { - - private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, - final Throwable t) { - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + level); - Weaver.callOriginal(); - } -} diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java index c360397096..2fb6ae28bf 100644 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.logbackclassic12" }) +@InstrumentationTestConfig(includePrefixes = { "ch.qos.logback" }) public class Logger_InstrumentationTest { private static final String CAPTURED = "This log message should be captured"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java index ef054e4384..ea554d22f3 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java @@ -131,6 +131,11 @@ public Insights getInsights() { return ServiceFactory.getServiceManager().getInsights(); } + @Override + public Insights getLogSender() { + return ServiceFactory.getServiceManager().getLogSenderService(); + } + @Override public boolean startAsyncActivity(Object activityContext) { return ServiceFactory.getAsyncTxService().startAsyncActivity(activityContext); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java index 154422182e..db9a21e644 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java @@ -79,6 +79,7 @@ public class DummyTransaction extends Transaction { private final Object lock = new Object(); private final Insights insights = new DummyInsights(); + private final Insights logEvents = new DummyLogEvents(); private final AgentConfig defaultConfig; private final TracerList tracerList = new TracerList(null, new DummySet()); private final TransactionTimer timer = new TransactionTimer(0); @@ -170,6 +171,11 @@ public Insights getInsightsData() { return insights; } + @Override + public Insights getLogEventData() { + return logEvents; + } + @Override public TransactionTracerConfig getTransactionTracerConfig() { return getAgentConfig().getTransactionTracerConfig(); @@ -651,6 +657,12 @@ public void recordCustomEvent(String eventType, Map attributes) { } } + static final class DummyLogEvents implements Insights { + @Override + public void recordCustomEvent(String eventType, Map attributes) { + } + } + static final class DummyCrossProcessState implements CrossProcessTransactionState { public static final CrossProcessTransactionState INSTANCE = new DummyCrossProcessState(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index bcf4b564e8..da50db2df9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -85,6 +85,7 @@ public void startHarvest(IRPMService rpmService) { public void startHarvestables(IRPMService rpmService, AgentConfig config) { Map eventHarvestConfig = config.getProperty(AgentConfigFactory.EVENT_HARVEST_CONFIG); Map spanHarvestConfig = config.getProperty(SERVER_SPAN_HARVEST_CONFIG); + // FIXME need to add log event harvest config if (eventHarvestConfig == null) { ServiceFactory.getStatsService().doStatsWork(StatsWorks.getIncrementCounterWork( MetricNames.SUPPORTABILITY_CONNECT_MISSING_EVENT_DATA, 1)); @@ -95,13 +96,21 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { int maxSamplesStored = tracker.harvestable.getMaxSamplesStored(); long reportPeriodInMillis = HarvestServiceImpl.REPORTING_PERIOD_IN_MILLISECONDS; boolean isSpanEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(SPAN_EVENT_DATA); + boolean isLogEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(LOG_EVENT_DATA); // The event_harvest_config received from server-side during the connect lifecycle contains config for error_event_data, analytic_event_data, and custom_event_data if (eventHarvestConfig != null && !isSpanEventEndpoint) { Agent.LOG.log(Level.FINE, "event_harvest_config from collector is: {0} samples stored for {1}", maxSamplesStored, tracker.harvestable.getEndpointMethodName()); Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); - Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); + // TODO set harvest_limits for log_event_data endpoint + Long harvestLimit; + // TODO THIS IS A HACK! Real limit for log_event_data endpoint should come from server side + if (isLogEventEndpoint) { + harvestLimit = 1000L; + } else { + harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); + } if (harvestLimit != null) { maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java index dec046d9b3..7fa6a8150c 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java @@ -11,6 +11,7 @@ import com.newrelic.agent.errors.TracedError; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.service.Service; @@ -82,6 +83,8 @@ public interface IRPMService extends Service { void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + void sendErrorEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception; void sendSpanEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java index ecf7ce8668..051b6aa6d7 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java @@ -330,6 +330,8 @@ public class MetricNames { // insights public static final String SUPPORTABILITY_API_RECORD_CUSTOM_EVENT = "RecordCustomEvent"; + public static final String SUPPORTABILITY_API_RECORD_LOG_EVENT = "RecordLogEvent"; + // attributes public static final String SUPPORTABILITY_API_ADD_CUSTOM_PARAMETER = "AddCustomParameter"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java index eebf6ae3ad..326c207320 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java @@ -25,6 +25,7 @@ import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.normalization.Normalizer; import com.newrelic.agent.profile.ProfileData; @@ -141,6 +142,7 @@ private Boolean getAndLogHighSecurity(AgentConfig config) { private void addHarvestablesToServices() { ServiceFactory.getServiceManager().getInsights().addHarvestableToService(appName); + ServiceFactory.getServiceManager().getLogSenderService().addHarvestableToService(appName); ServiceFactory.getTransactionEventsService().addHarvestableToService(appName); errorService.addHarvestableToService(); ServiceFactory.getSpanEventService().addHarvestableToService(appName); @@ -529,6 +531,27 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, final C } } + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception { + Agent.LOG.log(Level.FINE, "Sending {0} Log Sender event(s)", events.size()); + try { + sendLogEventsSyncRestart(reservoirSize, eventsSeen, events); + } catch (HttpError e) { + // We don't want to resend the data for certain response codes, retry for all others + if (e.isRetryableError()) { + throw e; + } + } catch (ForceRestartException e) { + logForceRestartException(e); + reconnectAsync(); + throw e; + } catch (ForceDisconnectException e) { + logForceDisconnectException(e); + shutdownAsync(); + throw e; + } + } + private void sendSpanEventsSyncRestart(int reservoirSize, int eventsSeen, final Collection events) throws Exception { try { dataSender.sendSpanEvents(reservoirSize, eventsSeen, events); @@ -571,6 +594,17 @@ private void sendCustomAnalyticsEventsSyncRestart(int reservoirSize, int eventsS } } + private void sendLogEventsSyncRestart(int reservoirSize, int eventsSeen, final Collection events) + throws Exception { + try { + dataSender.sendLogEvents(reservoirSize, eventsSeen, events); + } catch (ForceRestartException e) { + logForceRestartException(e); + reconnectSync(); + dataSender.sendLogEvents(reservoirSize, eventsSeen, events); + } + } + @Override public void sendErrorEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception { Agent.LOG.log(Level.FINE, "Sending {0} error event(s)", events.size()); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java index 42779deb13..f1ae36373a 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java @@ -191,6 +191,9 @@ public class Transaction { // Insights events added by the user during this transaction private final AtomicReference insights; + // Insights events added by the user during this transaction + private final AtomicReference logEvents; + // contains all work currently running private final Map runningChildren; @@ -448,6 +451,7 @@ protected Transaction() { userAttributes = new LazyMapImpl<>(factory); errorAttributes = new LazyMapImpl<>(factory); insights = new AtomicReference<>(); + logEvents = new AtomicReference<>(); runningChildren = new LazyMapImpl<>(factory); activeTokensCache = new AtomicReference<>(); activeCount = new AtomicInteger(0); @@ -613,6 +617,16 @@ public Insights getInsightsData() { return insightsData; } + public Insights getLogEventData() { + Insights logEventData = logEvents.get(); + if (logEventData == null) { + AgentConfig defaultConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); + logEvents.compareAndSet(null, ServiceFactory.getServiceManager().getLogSenderService().getTransactionInsights(defaultConfig)); + logEventData = logEvents.get(); + } + return logEventData; + } + public TransactionTracerConfig getTransactionTracerConfig() { if (dispatcher == null) { return getAgentConfig().getTransactionTracerConfig(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java b/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java index 7d17576dd2..940809103f 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java @@ -48,6 +48,10 @@ public Insights getInsightsData() { return tx.getInsightsData(); } + public Insights getLogEventData() { + return tx.getLogEventData(); + } + public Dispatcher getDispatcher() { return tx.getDispatcher(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java index 04c84df569..5c30b09c7a 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java @@ -172,6 +172,11 @@ public interface AgentConfig extends com.newrelic.api.agent.Config, DataSenderCo */ InsightsConfig getInsightsConfig(); + /** + * Get the Log Sender configuration. + */ + LogSenderConfig getLogSenderConfig(); + /** * Get the attributes configuration. */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java index 61680de690..1911647aad 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java @@ -115,6 +115,7 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig { public static final String JAR_COLLECTOR = "jar_collector"; public static final String JMX = "jmx"; public static final String JFR = "jfr"; + public static final String LOG_SENDING = "log_sending"; public static final String OPEN_TRACING = "open_tracing"; public static final String REINSTRUMENT = "reinstrument"; public static final String SLOW_SQL = "slow_sql"; @@ -253,6 +254,7 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig { private final ExternalTracerConfig externalTracerConfig; private final InfiniteTracingConfig infiniteTracingConfig; private final InsightsConfig insightsConfig; + private final LogSenderConfig logSenderConfig; private final Config instrumentationConfig; private final JarCollectorConfig jarCollectorConfig; private final JfrConfig jfrConfig; @@ -352,6 +354,7 @@ private AgentConfigImpl(Map props) { jmxConfig = initJmxConfig(); jarCollectorConfig = initJarCollectorConfig(); insightsConfig = initInsightsConfig(); + logSenderConfig = initLogSenderConfig(); infiniteTracingConfig = initInfiniteTracingConfig(autoAppNamingEnabled); attributesConfig = initAttributesConfig(); reinstrumentConfig = initReinstrumentConfig(); @@ -732,10 +735,15 @@ private JarCollectorConfig initJarCollectorConfig() { } private InsightsConfig initInsightsConfig() { - Map props = nestedProps(CUSTOM_INSIGHT_EVENTS); + Map props = nestedProps(LOG_SENDING); return InsightsConfigImpl.createInsightsConfig(props, highSecurity); } + private LogSenderConfig initLogSenderConfig() { + Map props = nestedProps(CUSTOM_INSIGHT_EVENTS); + return LogSenderConfigImpl.createLogSenderConfig(props, highSecurity); + } + private AttributesConfig initAttributesConfig() { Map props = nestedProps(ATTRIBUTES); return AttributesConfigImpl.createAttributesConfig(props); @@ -1183,6 +1191,11 @@ public InsightsConfig getInsightsConfig() { return insightsConfig; } + @Override + public LogSenderConfig getLogSenderConfig() { + return logSenderConfig; + } + @Override public AttributesConfig getAttributesConfig() { return attributesConfig; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java new file mode 100644 index 0000000000..6c197b2fc2 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java @@ -0,0 +1,19 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +/** + * Configuration got LogSenderService + */ +public interface LogSenderConfig { + + boolean isEnabled(); + + int getMaxSamplesStored(); + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java new file mode 100644 index 0000000000..cf12df56a3 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java @@ -0,0 +1,59 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import java.util.Collections; +import java.util.Map; + +public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { + public static final String MAX_SAMPLES_STORED_PROP = "max_samples_stored"; + public static final int DEFAULT_MAX_SAMPLES_STORED = 10000; + public static final String ENABLED_PROP = "enabled"; + public static final boolean DEFAULT_ENABLED = true; + public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config.log_sending."; + public static final String ENABLED = SYSTEM_PROPERTY_ROOT + ENABLED_PROP; +// public static final String COLLECT_CUSTOM_EVENTS = "collect_custom_events"; + + public final int maxSamplesStored; + public final boolean isEnabled; + + public LogSenderConfigImpl(Map pProps, boolean highSecurity) { + super(pProps, SYSTEM_PROPERTY_ROOT); + maxSamplesStored = getProperty(MAX_SAMPLES_STORED_PROP, DEFAULT_MAX_SAMPLES_STORED); + isEnabled = !highSecurity && initEnabled(); + } + + public boolean initEnabled() { + boolean storedMoreThan0 = maxSamplesStored > 0; + Boolean configEnabled = getProperty(ENABLED_PROP, DEFAULT_ENABLED); +// /* +// * "collect_analytics_events" is the property which comes down from the server. This gets mapped to +// * transaction_events.collect_analytics_events in AgentConfigFactory.mergeServerData() +// */ +// Boolean featureGateEnabled = getProperty(COLLECT_CUSTOM_EVENTS, DEFAULT_ENABLED); +// return storedMoreThan0 && configEnabled && featureGateEnabled; + return storedMoreThan0 && configEnabled; + } + + // TODO ignore highSecurity for the time being + static LogSenderConfigImpl createLogSenderConfig(Map settings, boolean highSecurity) { + if (settings == null) { + settings = Collections.emptyMap(); + } + return new LogSenderConfigImpl(settings, highSecurity); + } + + public boolean isEnabled() { + return isEnabled; + } + + public int getMaxSamplesStored() { + return maxSamplesStored; + } + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManager.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManager.java index d4cdd38ab5..1f2656d2cb 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManager.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManager.java @@ -36,6 +36,7 @@ import com.newrelic.agent.service.analytics.SpanEventsService; import com.newrelic.agent.service.analytics.TransactionEventsService; import com.newrelic.agent.service.async.AsyncTransactionService; +import com.newrelic.agent.service.logging.LogSenderService; import com.newrelic.agent.service.module.JarCollectorService; import com.newrelic.agent.sql.SqlTraceService; import com.newrelic.agent.stats.StatsService; @@ -109,6 +110,8 @@ public interface ServiceManager extends Service { InsightsService getInsights(); + LogSenderService getLogSenderService(); + AsyncTransactionService getAsyncTxService(); CircuitBreakerService getCircuitBreakerService(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManagerImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManagerImpl.java index 4b6cf8337b..aeaf949d61 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManagerImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceManagerImpl.java @@ -45,6 +45,8 @@ import com.newrelic.agent.samplers.SamplerServiceImpl; import com.newrelic.agent.service.analytics.*; import com.newrelic.agent.service.async.AsyncTransactionService; +import com.newrelic.agent.service.logging.LogSenderService; +import com.newrelic.agent.service.logging.LogSenderServiceImpl; import com.newrelic.agent.service.module.*; import com.newrelic.agent.sql.SqlTraceService; import com.newrelic.agent.sql.SqlTraceServiceImpl; @@ -113,6 +115,7 @@ public class ServiceManagerImpl extends AbstractService implements ServiceManage private volatile AttributesService attsService; private volatile UtilizationService utilizationService; private volatile InsightsService insightsService; + private volatile LogSenderService logSenderService; private volatile AsyncTransactionService asyncTxService; private volatile CircuitBreakerService circuitBreakerService; private volatile DistributedTraceServiceImpl distributedTraceService; @@ -234,6 +237,7 @@ protected synchronized void doStart() throws Exception { remoteInstrumentationService = new RemoteInstrumentationServiceImpl(); attsService = new AttributesService(); insightsService = new InsightsServiceImpl(); + logSenderService = new LogSenderServiceImpl(); spanEventsService = SpanEventsServiceFactory.builder() .configService(configService) .rpmServiceManager(rpmServiceManager) @@ -276,6 +280,7 @@ protected synchronized void doStart() throws Exception { remoteInstrumentationService.start(); attsService.start(); insightsService.start(); + logSenderService.start(); circuitBreakerService.start(); distributedTraceService.start(); spanEventsService.start(); @@ -310,6 +315,7 @@ private InfiniteTracing buildInfiniteTracing(ConfigService configService) { @Override protected synchronized void doStop() throws Exception { insightsService.stop(); + logSenderService.stop(); circuitBreakerService.stop(); remoteInstrumentationService.stop(); configService.stop(); @@ -575,6 +581,11 @@ public InsightsService getInsights() { return insightsService; } + @Override + public LogSenderService getLogSenderService() { + return logSenderService; + } + @Override public CircuitBreakerService getCircuitBreakerService() { return circuitBreakerService; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java new file mode 100644 index 0000000000..e102f10932 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.service.logging; + +import com.newrelic.agent.Harvestable; +import com.newrelic.agent.service.ServiceFactory; +import com.newrelic.agent.transport.CollectorMethods; + +public class LogSenderHarvestableImpl extends Harvestable { + + LogSenderHarvestableImpl(LogSenderServiceImpl logSenderService, String appName) { + super(logSenderService, appName); + } + + /** + * Agent endpoint to send log data to. + * + * @return String representing the endpoint name + */ + @Override + public String getEndpointMethodName() { + return CollectorMethods.LOG_EVENT_DATA; + } + + /** + * Number of log sender events that can be stored. + * + * @return int for max samples stored + */ + @Override + public int getMaxSamplesStored() { + return ServiceFactory.getConfigService().getDefaultAgentConfig().getLogSenderConfig().getMaxSamplesStored(); + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java new file mode 100644 index 0000000000..826ca0bfae --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java @@ -0,0 +1,43 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.service.logging; + +import com.newrelic.agent.config.AgentConfig; +import com.newrelic.agent.model.LogEvent; +import com.newrelic.agent.service.EventService; +import com.newrelic.api.agent.Insights; + +/** + * LogSenderService interface + * + * Extending Insights makes the recordCustomEvent(...) API available to implementing classes + */ +public interface LogSenderService extends EventService, Insights { + + /** + * Returns an insights instance used to track events created during a transaction. The events will be reported to + * the Transaction's application, or to the default application if not in a transaction. + * + */ + Insights getTransactionInsights(AgentConfig config); + + /** + * Store event into Reservoir following usual sampling using the given appName. Preference should be given to + * storing the event in TransactionInsights instead of this. + * @param appName application name + * @param event log event + */ + void storeEvent(String appName, LogEvent event); + + /** + * + * @param appName application name + */ + void addHarvestableToService(String appName); + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java new file mode 100644 index 0000000000..3d32ac6f63 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -0,0 +1,486 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.service.logging; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.annotations.VisibleForTesting; +import com.newrelic.agent.Agent; +import com.newrelic.agent.ExtendedTransactionListener; +import com.newrelic.agent.Harvestable; +import com.newrelic.agent.MetricNames; +import com.newrelic.agent.Transaction; +import com.newrelic.agent.TransactionData; +import com.newrelic.agent.attributes.AttributeSender; +import com.newrelic.agent.attributes.AttributeValidator; +import com.newrelic.agent.config.AgentConfig; +import com.newrelic.agent.config.AgentConfigListener; +import com.newrelic.agent.model.AnalyticsEvent; +import com.newrelic.agent.model.LogEvent; +import com.newrelic.agent.service.AbstractService; +import com.newrelic.agent.service.ServiceFactory; +import com.newrelic.agent.service.analytics.DistributedSamplingPriorityQueue; +import com.newrelic.agent.stats.StatsEngine; +import com.newrelic.agent.stats.StatsWork; +import com.newrelic.agent.stats.TransactionStats; +import com.newrelic.agent.tracing.DistributedTraceServiceImpl; +import com.newrelic.agent.transport.HttpError; +import com.newrelic.api.agent.Insights; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; + +public class LogSenderServiceImpl extends AbstractService implements LogSenderService { + // Whether the service as a whole is enabled. Disabling shuts down all analytic events for transactions. + private volatile boolean enabled; + // Key is the app name, value is if it is enabled - should be a limited number of names + private final ConcurrentMap isEnabledForApp = new ConcurrentHashMap<>(); + /* + * Number of events in the reservoir sampling buffer per-app. All apps get the same value. Synthetics are buffered + * separately per app using a deterministic algorithm. + */ + private volatile int maxSamplesStored; + + // Key is app name, value is collection of per-transaction analytic events for next harvest for that app. + private final ConcurrentHashMap> reservoirForApp = new ConcurrentHashMap<>(); + + private static final LoadingCache stringCache = Caffeine.newBuilder().maximumSize(1000) + .expireAfterAccess(70, TimeUnit.SECONDS).executor(Runnable::run).build(key -> key); + + // TODO it's not clear that log sender events should be tied to transactions in any way + protected final ExtendedTransactionListener transactionListener = new ExtendedTransactionListener() { + + @Override + public void dispatcherTransactionStarted(Transaction transaction) { + } + + @Override + public void dispatcherTransactionFinished(TransactionData transactionData, TransactionStats transactionStats) { + // FIXME not sure this is a great idea to store log events for the duration of a transaction... + TransactionInsights data = (TransactionInsights) transactionData.getLogEventData(); + storeEvents(transactionData.getApplicationName(), transactionData.getPriority(), data.events); + } + + @Override + public void dispatcherTransactionCancelled(Transaction transaction) { + // FIXME not sure this is a great idea to store log events for the duration of a transaction... + // Even if the transaction is cancelled we still want to send up any events that were held in it + TransactionInsights data = (TransactionInsights) transaction.getLogEventData(); + storeEvents(transaction.getApplicationName(), transaction.getPriority(), data.events); + } + + }; + + protected final AgentConfigListener configListener = new AgentConfigListener() { + @Override + public void configChanged(String appName, AgentConfig agentConfig) { + // if the config has changed for the app, just remove it and regenerate enabled next transaction + isEnabledForApp.remove(appName); + enabled = agentConfig.getLogSenderConfig().isEnabled(); + } + }; + + private List harvestables = new ArrayList<>(); + + public LogSenderServiceImpl() { + super(LogSenderServiceImpl.class.getSimpleName()); + AgentConfig config = ServiceFactory.getConfigService().getDefaultAgentConfig(); + maxSamplesStored = config.getLogSenderConfig().getMaxSamplesStored(); + enabled = config.getLogSenderConfig().isEnabled(); + isEnabledForApp.put(config.getApplicationName(), enabled); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + protected void doStart() throws Exception { + // TODO it's not clear that log sender events should be tied to transactions in any way + ServiceFactory.getTransactionService().addTransactionListener(transactionListener); + ServiceFactory.getConfigService().addIAgentConfigListener(configListener); + } + + @Override + protected void doStop() throws Exception { + removeHarvestables(); + // TODO it's not clear that log sender events should be tied to transactions in any way + ServiceFactory.getTransactionService().removeTransactionListener(transactionListener); + ServiceFactory.getConfigService().removeIAgentConfigListener(configListener); + reservoirForApp.clear(); + isEnabledForApp.clear(); + stringCache.invalidateAll(); + } + + private void removeHarvestables() { + for (Harvestable harvestable : harvestables) { + ServiceFactory.getHarvestService().removeHarvestable(harvestable); + } + } + + @Override + public void recordCustomEvent(String eventType, Map attributes) { + if (logSenderEventsDisabled(eventType)) { + return; + } + + if (AnalyticsEvent.isValidType(eventType)) { + Transaction transaction = ServiceFactory.getTransactionService().getTransaction(false); + // FIXME perhaps ignore transaction status and just always send log events... + // what is the benefit of storing them on the transaction? Sampling maybe? + if (transaction == null || !transaction.isInProgress() || transaction.isIgnore()) { + String applicationName = ServiceFactory.getRPMService().getApplicationName(); + if (transaction != null && transaction.getApplicationName() != null) { + applicationName = transaction.getApplicationName(); + } + AgentConfig agentConfig = ServiceFactory.getConfigService().getAgentConfig(applicationName); + if (!getIsEnabledForApp(agentConfig, applicationName)) { + reservoirForApp.remove(applicationName); + return; + } + storeEvent(applicationName, eventType, attributes); + } else { + // FIXME not sure this is a great idea to store log events for the duration of a transaction... + transaction.getLogEventData().recordCustomEvent(eventType, attributes); + } + MetricNames.recordApiSupportabilityMetric(MetricNames.SUPPORTABILITY_API_RECORD_LOG_EVENT); + } else { + Agent.LOG.log(Level.WARNING, "Custom event with invalid type of {0} was reported but ignored." + + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); + } + } + + private void storeEvents(String appName, float priority, Collection events) { + if (events.size() > 0) { + DistributedSamplingPriorityQueue eventList = getReservoir(appName); + for (LogEvent event : events) { + // Set "priority" on LogSenderEvent based on priority value from Transaction + event.setPriority(priority); + eventList.add(event); + } + } + } + + public void addHarvestableToService(String appName) { + Harvestable harvestable = new LogSenderHarvestableImpl(this, appName); + ServiceFactory.getHarvestService().addHarvestable(harvestable); + harvestables.add(harvestable); + } + + public int getMaxSamplesStored() { + return maxSamplesStored; + } + + public void setMaxSamplesStored(int maxSamplesStored) { + this.maxSamplesStored = maxSamplesStored; + } + + public void clearReservoir() { + reservoirForApp.clear(); + } + + public void clearReservoir(String appName) { + DistributedSamplingPriorityQueue reservoir = reservoirForApp.get(appName); + if (reservoir != null) { + reservoir.clear(); + } + } + + @VisibleForTesting + void configureHarvestables(long reportPeriodInMillis, int maxSamplesStored) { + for (Harvestable h : harvestables) { + h.configure(reportPeriodInMillis, maxSamplesStored); + } + } + + @VisibleForTesting + public void harvestHarvestables() { + for (Harvestable h : harvestables) { + h.harvest(); + } + } + + public void harvestPendingEvents() { + // harvest pending events + for (String appName : reservoirForApp.keySet()) { + harvestEvents(appName); + } + } + + @Override + public void storeEvent(String appName, LogEvent event) { + if (logSenderEventsDisabled(event.getType())) { + return; + } + + DistributedSamplingPriorityQueue eventList = getReservoir(appName); + eventList.add(event); + Agent.LOG.finest(MessageFormat.format("Added Custom Event of type {0}", event.getType())); + } + + private void storeEvent(String appName, String eventType, Map attributes) { + if (logSenderEventsDisabled(eventType)) { + return; + } + + DistributedSamplingPriorityQueue eventList = getReservoir(appName); + eventList.add(createValidatedEvent(eventType, attributes)); + Agent.LOG.finest(MessageFormat.format("Added Custom Event of type {0}", eventType)); + } + + private boolean logSenderEventsDisabled(String eventType) { + if (!enabled) { + // TODO just ignore high security for now + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); + } else { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", eventType); + } + + return true; // Log Sender events are disabled + } + + return false; // Log Sender events are enabled + } + + @VisibleForTesting + public DistributedSamplingPriorityQueue getReservoir(String appName) { + DistributedSamplingPriorityQueue result = reservoirForApp.get(appName); + while (result == null) { + // I don't think this loop can actually execute more than once, but it's prudent to assume it can. + reservoirForApp.putIfAbsent(appName, new DistributedSamplingPriorityQueue(appName, "Log Sender Service", maxSamplesStored)); + result = reservoirForApp.get(appName); + } + return result; + } + + public void harvestEvents(final String appName) { + if (!getIsEnabledForApp(ServiceFactory.getConfigService().getAgentConfig(appName), appName)) { + reservoirForApp.remove(appName); + return; + } + if (maxSamplesStored <= 0) { + clearReservoir(appName); + return; + } + + long startTimeInNanos = System.nanoTime(); + + final DistributedSamplingPriorityQueue reservoir = this.reservoirForApp.put(appName, + new DistributedSamplingPriorityQueue<>(appName, "Log Sender Service", maxSamplesStored)); + + if (reservoir != null && reservoir.size() > 0) { + try { + // TODO actual sending of events + + ServiceFactory.getRPMServiceManager() + .getOrCreateRPMService(appName) + .sendLogEvents(maxSamplesStored, reservoir.getNumberOfTries(), Collections.unmodifiableList(reservoir.asList())); + final long durationInNanos = System.nanoTime() - startTimeInNanos; + ServiceFactory.getStatsService().doStatsWork(new StatsWork() { + @Override + public void doWork(StatsEngine statsEngine) { + recordSupportabilityMetrics(statsEngine, durationInNanos, reservoir); + } + + @Override + public String getAppName() { + return appName; + } + }); + + if (reservoir.size() < reservoir.getNumberOfTries()) { + int dropped = reservoir.getNumberOfTries() - reservoir.size(); + Agent.LOG.log(Level.FINE, "Dropped {0} custom events out of {1}.", dropped, reservoir.getNumberOfTries()); + } + } catch (HttpError e) { + if (!e.discardHarvestData()) { + Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be included in the next harvest.", e); + // Save unsent data by merging it with current data using reservoir algorithm + DistributedSamplingPriorityQueue currentReservoir = reservoirForApp.get(appName); + currentReservoir.retryAll(reservoir); + } else { + // discard harvest data + reservoir.clear(); + Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be dropped.", e); + } + } catch (Exception e) { + // discard harvest data + reservoir.clear(); + Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be dropped.", e); + } + } + } + + @Override + public String getEventHarvestIntervalMetric() { + return MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_INTERVAL; + } + + @Override + public String getReportPeriodInSecondsMetric() { + return MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_REPORT_PERIOD_IN_SECONDS; + } + + @Override + public String getEventHarvestLimitMetric() { + return MetricNames.SUPPORTABILITY_CUSTOM_EVENT_DATA_HARVEST_LIMIT; + } + + private void recordSupportabilityMetrics(StatsEngine statsEngine, long durationInNanoseconds, + DistributedSamplingPriorityQueue reservoir) { + statsEngine.getStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SENT) + .incrementCallCount(reservoir.size()); + statsEngine.getStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SEEN) + .incrementCallCount(reservoir.getNumberOfTries()); + statsEngine.getResponseTimeStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_TRANSMIT) + .recordResponseTime(durationInNanoseconds, TimeUnit.NANOSECONDS); + } + + private boolean getIsEnabledForApp(AgentConfig config, String currentAppName) { + Boolean appEnabled = currentAppName == null ? null : isEnabledForApp.get(currentAppName); + if (appEnabled == null) { + appEnabled = config.getLogSenderConfig().isEnabled(); + isEnabledForApp.put(currentAppName, appEnabled); + } + return appEnabled; + } + + /** + * We put Strings that occur in events in a map so that we're only ever holding a reference to one byte array for + * any given string. It's basically like interning the string without using a global map. + * + * @param value the string to "intern" + * @return the interned string + */ + private static String mapInternString(String value) { + // Note that the interning occurs on the *input* to the validation code. If the validation code truncates or + // otherwise replaces the "interned" string, the new string will not be "interned" by this cache. See the + // comment below for more information. + return stringCache.get(value); + } + + private static LogEvent createValidatedEvent(String eventType, Map attributes) { + Map userAttributes = new HashMap<>(attributes.size()); + LogEvent event = new LogEvent(mapInternString(eventType), System.currentTimeMillis(), userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); + + // Now add the attributes from the argument map to the event using an AttributeSender. + // An AttributeSender is the way to reuse all the existing attribute validations. We + // also locally "intern" Strings because we anticipate a lot of reuse of the keys and, + // possibly, the values. But there's an interaction: if the key or value is chopped + // within the attribute sender, the modified value won't be "interned" in our map. + + // FIXME probably don't need this AttributeSender for LogSenderEvents + AttributeSender sender = new LogSenderEventAttributeSender(userAttributes); + final String method = "add log sender event attribute"; + + for (Map.Entry entry : attributes.entrySet()) { + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + // key or value is null, skip it with a log message and iterate to next entry in attributes.entrySet() + if (key == null || value == null) { + Agent.LOG.log(Level.WARNING, "Log Sender event with invalid attributes key or value of null was reported for a transaction but ignored." + + " Each key should be a String and each value should be a String, Number, or Boolean."); + continue; + } + + mapInternString(key); + + if (value instanceof String) { + sender.addAttribute(key, mapInternString((String) value), method); + } else if (value instanceof Number) { + sender.addAttribute(key, (Number) value, method); + } else if (value instanceof Boolean) { + sender.addAttribute(key, (Boolean) value, method); + } else { + // Java Agent specific - toString the value. This allows for e.g. enums as arguments. + sender.addAttribute(key, mapInternString(value.toString()), method); + } + } + + return event; + } + + private static class LogSenderEventAttributeSender extends AttributeSender { + + private static final String ATTRIBUTE_TYPE = "custom"; + + private final Map userAttributes; + + public LogSenderEventAttributeSender(Map userAttributes) { + super(new AttributeValidator(ATTRIBUTE_TYPE)); + this.userAttributes = userAttributes; + setTransactional(false); + } + + @Override + protected String getAttributeType() { + return ATTRIBUTE_TYPE; + } + + @Override + protected Map getAttributeMap() { + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { + return userAttributes; + } + return null; + } + } + + @Override + public Insights getTransactionInsights(AgentConfig config) { + return new TransactionInsights(config); + } + + public static final class TransactionInsights implements Insights { + final LinkedBlockingQueue events; + + TransactionInsights(AgentConfig config) { + int maxSamplesStored = config.getLogSenderConfig().getMaxSamplesStored(); + events = new LinkedBlockingQueue<>(maxSamplesStored); + } + + @Override + public void recordCustomEvent(String eventType, Map attributes) { + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); + return; + } + + if (AnalyticsEvent.isValidType(eventType)) { + LogEvent event = createValidatedEvent(eventType, attributes); + if (events.offer(event)) { + Agent.LOG.finest(MessageFormat.format("Added event of type {0} in Transaction.", eventType)); + } else { + // Too many events are cached on the transaction, send directly to the reservoir. + String applicationName = ServiceFactory.getRPMService().getApplicationName(); + ServiceFactory.getServiceManager().getLogSenderService().storeEvent(applicationName, event); + } + } else { + Agent.LOG.log(Level.WARNING, "LogSender event with invalid type of {0} was reported for a transaction but ignored." + + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); + } + } + + public List getEventsForTesting() { + return new ArrayList<>(events); + } + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/CollectorMethods.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/CollectorMethods.java index 1483cc7299..701c45cd21 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/CollectorMethods.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/CollectorMethods.java @@ -19,6 +19,7 @@ public class CollectorMethods { public static final String ANALYTIC_EVENT_DATA = "analytic_event_data"; public static final String SPAN_EVENT_DATA = "span_event_data"; public static final String CUSTOM_EVENT_DATA = "custom_event_data"; + public static final String LOG_EVENT_DATA = "log_event_data"; public static final String UPDATE_LOADED_MODULES = "update_loaded_modules"; public static final String SHUTDOWN = "shutdown"; public static final String SQL_TRACE_DATA = "sql_trace_data"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java index 363b4c0c3d..2b0484d958 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java @@ -12,6 +12,7 @@ import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.sql.SqlTrace; @@ -44,6 +45,11 @@ public interface DataSender { */ void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + /** + * Send non-aggregated Log events + */ + void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + /** * Send non-aggregated span events */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index e46e721f1a..62b3fd511e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -24,6 +24,7 @@ import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.service.ServiceFactory; @@ -331,6 +332,12 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect sendAnalyticEventsForReservoir(CollectorMethods.CUSTOM_EVENT_DATA, compressedEncoding, reservoirSize, eventsSeen, events); } + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + // TODO use sendAnalyticEventsForReservoir? Or create new method to handle MELT format for log_event_data endpoint??? + sendLogEventsForReservoir(CollectorMethods.LOG_EVENT_DATA, compressedEncoding, reservoirSize, eventsSeen, events); + } + @Override public void sendSpanEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { sendAnalyticEventsForReservoir(CollectorMethods.SPAN_EVENT_DATA, compressedEncoding, reservoirSize, eventsSeen, events); @@ -354,6 +361,26 @@ private void sendAnalyticEventsForR invokeRunId(method, encoding, runId, params); } + private void sendLogEventsForReservoir(String method, String encoding, int reservoirSize, int eventsSeen, + Collection events) throws Exception { + Object runId = agentRunId; + if (runId == NO_AGENT_RUN_ID || events.isEmpty()) { + return; + } + InitialSizedJsonArray params = new InitialSizedJsonArray(3); + params.add(runId); + + JSONObject metadata = new JSONObject(); + metadata.put("reservoir_size", reservoirSize); + metadata.put("events_seen", eventsSeen); + // TODO if this is used for log_event_data a conditional check will be needed for other attributes specific to log sender events + params.add(metadata); + + // TODO reshape the format of the events to better fit log data + params.add(events); + invokeRunId(method, encoding, runId, params); + } + @Override public void sendMetricData(long beginTimeMillis, long endTimeMillis, List metricData) throws Exception { Object runId = agentRunId; diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java b/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java index 2386a27ffe..081a59b236 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java @@ -13,6 +13,7 @@ import com.newrelic.agent.logging.IAgentLogger; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.service.analytics.TransactionEvent; @@ -162,6 +163,10 @@ public void sendAnalyticsEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { } + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + } + @Override public void sendErrorEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java index 7ef7496927..6526581295 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java @@ -12,6 +12,7 @@ import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.ProfileData; import com.newrelic.agent.sql.SqlTrace; @@ -148,4 +149,8 @@ public void sendSpanEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception { } + + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + } } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java index dc27d80dc9..66e30f6b5d 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java @@ -12,6 +12,7 @@ import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.CustomInsightsEvent; import com.newrelic.agent.model.ErrorEvent; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.SpanEvent; import com.newrelic.agent.profile.IProfile; import com.newrelic.agent.profile.ProfileData; @@ -47,6 +48,7 @@ public class MockRPMService extends BaseRPMService { private final AtomicInteger transactionEventsSeen = new AtomicInteger(0); private final AtomicInteger spanEventsSeen = new AtomicInteger(0); private final AtomicInteger customEventsSeen = new AtomicInteger(0); + private final AtomicInteger logSenderEventsSeen = new AtomicInteger(0); private final AtomicInteger errorEventsSeen = new AtomicInteger(0); private final AtomicInteger errorTracesSeen = new AtomicInteger(0); private final AtomicInteger transactionTracesSeen = new AtomicInteger(0); @@ -222,6 +224,12 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect this.customEventsSeen.addAndGet(eventsSeen); } + @Override + public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + this.events.addAll(events); + this.logSenderEventsSeen.addAndGet(eventsSeen); + } + @Override public void sendErrorEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { this.events.addAll(events); @@ -261,6 +269,10 @@ public int getCustomEventsSeen() { return customEventsSeen.get(); } + public int getLogSenderEventsSeen() { + return logSenderEventsSeen.get(); + } + public int getErrorEventsSeen() { return errorEventsSeen.get(); } @@ -280,6 +292,7 @@ public void clearEvents() { spanEventsSeen.set(0); transactionEventsSeen.set(0); customEventsSeen.set(0); + logSenderEventsSeen.set(0); errorTracesSeen.set(0); transactionTracesSeen.set(0); } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockServiceManager.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockServiceManager.java index dbc2e742ef..6523a625f2 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockServiceManager.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockServiceManager.java @@ -38,6 +38,8 @@ import com.newrelic.agent.service.ServiceManager; import com.newrelic.agent.service.analytics.*; import com.newrelic.agent.service.async.AsyncTransactionService; +import com.newrelic.agent.service.logging.LogSenderService; +import com.newrelic.agent.service.logging.LogSenderServiceImpl; import com.newrelic.agent.service.module.JarCollectorService; import com.newrelic.agent.sql.SqlTraceService; import com.newrelic.agent.stats.StatsService; @@ -85,6 +87,7 @@ public class MockServiceManager extends AbstractService implements ServiceManage private volatile SpanEventsService spanEventsService; private volatile SourceLanguageService sourceLanguageService; private volatile InsightsService insights; + private volatile LogSenderService logSenderService; private volatile ExpirationService expirationService; public MockServiceManager() { @@ -144,6 +147,7 @@ public MockServiceManager(ConfigService configService) { circuitBreakerService = new CircuitBreakerService(); spanEventsService = Mockito.mock(SpanEventsService.class); insights = Mockito.mock(InsightsServiceImpl.class); + logSenderService = Mockito.mock(LogSenderServiceImpl.class); } @Override @@ -593,6 +597,15 @@ public void setInsights(InsightsService service) { insights = service; } + @Override + public LogSenderService getLogSenderService() { + return logSenderService; + } + + public void setLogSenderService(LogSenderService service) { + logSenderService = service; + } + @Override public CircuitBreakerService getCircuitBreakerService() { return circuitBreakerService; diff --git a/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java b/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java index b785f3f3a7..0a8f0ab9a7 100644 --- a/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java +++ b/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java @@ -63,6 +63,15 @@ public interface Agent { */ Insights getInsights(); + /** + * Provides access to the LogSender custom events API. + * + * @return Object used to add custom events. + * @since 3.13.0 + */ + // FIXME this should probably be on the agent bridge instead of a public API + Insights getLogSender(); + /** * Provides access to the Trace Metadata API for details about the currently executing distributed trace. * diff --git a/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java b/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java index 8f26efc2f7..ed1d22a3b5 100644 --- a/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java +++ b/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java @@ -439,6 +439,11 @@ public Insights getInsights() { return INSIGHTS; } + @Override + public Insights getLogSender() { + return INSIGHTS; + } + @Override public TraceMetadata getTraceMetadata() { return TRACE_METADATA; From 4269995860d9816898bcbcb13f57189d74b4d9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Mon, 24 Jan 2022 20:50:21 +0100 Subject: [PATCH 10/96] feat: add instrumentation to JUL (Java Logging) --- .../test/newrelic/test/agent/LoggerTest.java | 136 ++++++++++++++++++ .../java.logging-jdk8/build.gradle | 25 ++++ .../jul/Logger_Instrumentation.java | 30 ++++ settings.gradle | 1 + 4 files changed, 192 insertions(+) create mode 100644 functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java create mode 100644 instrumentation/java.logging-jdk8/build.gradle create mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java diff --git a/functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java b/functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java new file mode 100644 index 0000000000..0a754f5e92 --- /dev/null +++ b/functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java @@ -0,0 +1,136 @@ +package test.newrelic.test.agent; + +import com.newrelic.agent.Transaction; +import com.newrelic.agent.stats.SimpleStatsEngine; +import com.newrelic.agent.stats.TransactionStats; +import com.newrelic.api.agent.Trace; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +public class LoggerTest { + + private static final String CAPTURED = "This log message should be captured"; + private static final String NOT_CAPTURED = "This message should NOT be captured"; + + @Before + public void setup() { + Transaction.clearTransaction(); + } + + @Trace(dispatcher = true) + @Test + public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { + // Given + final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + logger.setLevel(Level.INFO); + + // When + logger.finest(NOT_CAPTURED); + logger.finer(NOT_CAPTURED); + logger.fine(NOT_CAPTURED); + logger.config(NOT_CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.warning(CAPTURED); + logger.warning(CAPTURED); + logger.warning(CAPTURED); + logger.warning(CAPTURED); + logger.severe(CAPTURED); + + // Then + Map metrics = getLogMetricsCounts(); + Assert.assertEquals(8, (int) metrics.get("Logging/lines")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINEST")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINER")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINE")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/CONFIG")); + Assert.assertEquals(3, (int) metrics.get("Logging/lines/INFO")); + Assert.assertEquals(4, (int) metrics.get("Logging/lines/WARNING")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/SEVERE")); + } + + @Trace(dispatcher = true) + @Test + public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToFinest() { + // Given + final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + logger.setLevel(Level.FINEST); + + // When + logger.finest(CAPTURED); + logger.finer(CAPTURED); + logger.fine(CAPTURED); + logger.config(CAPTURED); + logger.info(CAPTURED); + logger.warning(CAPTURED); + logger.severe(CAPTURED); + + // Then + Map metrics = getLogMetricsCounts(); + Assert.assertEquals(7, (int) metrics.get("Logging/lines")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/FINEST")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/FINER")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/FINE")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/CONFIG")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/INFO")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/WARNING")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/SEVERE")); + } + + @Trace(dispatcher = true) + @Test + public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabledEvenLoggingLogRecordsDirectly() { + // Given + final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + logger.setLevel(Level.INFO); + + // When + logger.log(new LogRecord(Level.FINEST, NOT_CAPTURED)); + logger.log(new LogRecord(Level.FINER, NOT_CAPTURED)); + logger.log(new LogRecord(Level.FINE, NOT_CAPTURED)); + logger.log(new LogRecord(Level.CONFIG, NOT_CAPTURED)); + logger.log(new LogRecord(Level.INFO, CAPTURED)); + logger.log(new LogRecord(Level.INFO, CAPTURED)); + logger.log(new LogRecord(Level.INFO, CAPTURED)); + logger.log(new LogRecord(Level.WARNING, CAPTURED)); + logger.log(new LogRecord(Level.WARNING, CAPTURED)); + logger.log(new LogRecord(Level.WARNING, CAPTURED)); + logger.log(new LogRecord(Level.WARNING, CAPTURED)); + logger.log(new LogRecord(Level.SEVERE, CAPTURED)); + + // Then + Map metrics = getLogMetricsCounts(); + Assert.assertEquals(8, (int) metrics.get("Logging/lines")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINEST")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINER")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/FINE")); + Assert.assertEquals(0, (int) metrics.get("Logging/lines/CONFIG")); + Assert.assertEquals(3, (int) metrics.get("Logging/lines/INFO")); + Assert.assertEquals(4, (int) metrics.get("Logging/lines/WARNING")); + Assert.assertEquals(1, (int) metrics.get("Logging/lines/SEVERE")); + } + + private Map getLogMetricsCounts() { + Transaction transaction = Transaction.getTransaction(); + TransactionStats transactionStats = transaction.getTransactionActivity().getTransactionStats(); + SimpleStatsEngine engine = transactionStats.getUnscopedStats(); + final Map metrics = new HashMap<>(); + metrics.put("Logging/lines", engine.getStats("Logging/lines").getCallCount()); + metrics.put("Logging/lines/FINEST", engine.getStats("Logging/lines/FINEST").getCallCount()); + metrics.put("Logging/lines/FINER", engine.getStats("Logging/lines/FINER").getCallCount()); + metrics.put("Logging/lines/FINE", engine.getStats("Logging/lines/FINE").getCallCount()); + metrics.put("Logging/lines/CONFIG", engine.getStats("Logging/lines/CONFIG").getCallCount()); + metrics.put("Logging/lines/INFO", engine.getStats("Logging/lines/INFO").getCallCount()); + metrics.put("Logging/lines/WARNING", engine.getStats("Logging/lines/WARNING").getCallCount()); + metrics.put("Logging/lines/SEVERE", engine.getStats("Logging/lines/SEVERE").getCallCount()); + return metrics; + } +} diff --git a/instrumentation/java.logging-jdk8/build.gradle b/instrumentation/java.logging-jdk8/build.gradle new file mode 100644 index 0000000000..bd72ac879c --- /dev/null +++ b/instrumentation/java.logging-jdk8/build.gradle @@ -0,0 +1,25 @@ +dependencies { + implementation(project(":agent-bridge")) +} + +// This instrumentation module should not use the bootstrap classpath +compileJava.options.bootstrapClasspath = null + +jar { + manifest { attributes 'Implementation-Title': 'com.newrelic.instrumentation.java.logging-jdk8' } +} + +verifyInstrumentation { + verifyClasspath = false // We don't want to verify classpath since these are JDK classes +} + +site { + title 'Java Logging' + type 'Other' + versionOverride '[8,)' +} + +compileJava { + options.fork = true + options.bootstrapClasspath = null +} \ No newline at end of file diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java new file mode 100644 index 0000000000..3cadb1c0bd --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java @@ -0,0 +1,30 @@ +package com.nr.instrumentation.jul; + +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; + +import java.util.logging.Filter; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +@Weave(originalName = "java.util.logging.Logger") +public class Logger_Instrumentation { + + public Filter getFilter() { + return Weaver.callOriginal(); + } + + public boolean isLoggable(Level level) { + return Boolean.TRUE.equals(Weaver.callOriginal()); + } + + public void log(LogRecord record) { + if (isLoggable(record.getLevel()) && getFilter() == null || getFilter().isLoggable(record)) { + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + record.getLevel().toString()); + } + Weaver.callOriginal(); + } + +} diff --git a/settings.gradle b/settings.gradle index fcc3d303c7..bb64fb23ee 100644 --- a/settings.gradle +++ b/settings.gradle @@ -158,6 +158,7 @@ include 'instrumentation:hystrix-1.3.15' include 'instrumentation:hystrix-1.4' include 'instrumentation:java.completable-future-jdk8' include 'instrumentation:java.completable-future-jdk8u40' +include 'instrumentation:java.logging-jdk8' include 'instrumentation:java-io' include 'instrumentation:javax.xml' include 'instrumentation:jax-rs-1.0' From 23e21851c0e038ec76141f9b057c0609156f1634 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 25 Jan 2022 14:12:20 -0800 Subject: [PATCH 11/96] Updates to log4j instrumentation --- .../com/newrelic/agent/model/LogEvent.java | 2 +- .../log4j}/Category_Instrumentation.java | 2 +- .../log4j1/Category_InstrumentationTest.java | 2 +- .../instrumentation/log4j2/AgentUtil.java | 60 +++++++++++++++++++ .../log4j2/LoggerConfig_Instrumentation.java | 17 ------ .../config/LoggerConfig_Instrumentation.java | 31 ++++++++++ .../log4j/core/impl/Log4jLogEvent.java | 29 +++++++++ .../apache/logging/log4j/message/Message.java | 8 +++ .../logging/log4j/spi/AbstractLogger.java | 5 ++ .../LoggerConfig_InstrumentationTest.java | 2 +- .../agent/transport/DataSenderImpl.java | 3 +- 11 files changed, 139 insertions(+), 22 deletions(-) rename instrumentation/apache-log4j-1/src/main/java/{com/nr/agent/instrumentation/log4j1 => org/apache/log4j}/Category_Instrumentation.java (91%) create mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java delete mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java index 18a4ddcad3..e5dc133608 100644 --- a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -39,7 +39,7 @@ public float getPriority() { public void writeJSONString(Writer out) throws IOException { JSONObject intrinsics = new JSONObject(); intrinsics.put("type", getType()); - intrinsics.put("timestamp", getTimestamp()); +// intrinsics.put("timestamp", getTimestamp()); // this probably isn't needed as there is a timestamp added for the log event already JSONArray.writeJSONString(Arrays.asList(intrinsics, getMutableUserAttributes()), out); } diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java similarity index 91% rename from instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java rename to instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java index 4478c102ab..d7c8c05f9f 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/Category_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java @@ -1,4 +1,4 @@ -package com.nr.agent.instrumentation.log4j1; +package org.apache.log4j; import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.Weave; diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java index a9a973477a..9177cbd703 100644 --- a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java +++ b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java @@ -10,7 +10,7 @@ import org.junit.runner.RunWith; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.log4j1" }) +@InstrumentationTestConfig(includePrefixes = { "org.apache.log4j" }) public class Category_InstrumentationTest { private static final String CAPTURED = "This log message should be captured"; diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java new file mode 100644 index 0000000000..da81e0f2fc --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -0,0 +1,60 @@ +package com.nr.agent.instrumentation.log4j2; + +import com.newrelic.api.agent.NewRelic; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.time.Instant; +import org.apache.logging.log4j.message.Message; + +import java.util.HashMap; +import java.util.Map; + +public class AgentUtil { + + public static void reportNewRelicLogEvent(LogEvent event) { + final String EMPTY_STRING = ""; + + if (event != null) { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + HashMap logEventMap = new HashMap<>(agentLinkingMetadata); + + Message message = event.getMessage(); + logEventMap.put("message", message != null ? message.getFormattedMessage() : EMPTY_STRING); + logEventMap.put("timeStampTimeMillis", event.getTimeMillis()); + + Instant instant = event.getInstant(); + logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); + + Level level = event.getLevel(); + logEventMap.put("log.level", level != null ? level.name() : EMPTY_STRING); + logEventMap.put("logger.name", event.getLoggerName()); + logEventMap.put("class.name", event.getLoggerFqcn()); + + Throwable throwable = event.getThrown(); + logEventMap.put("throwable", throwable != null ? throwable.toString() : EMPTY_STRING); + + NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); + } + } + +// public static Map createLogEventAttributesMap(LogEvent event) { +// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// // TODO omit attributes that don't have values +// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} +// return linkingMetadata; +// } +// +// public static Map getLinkingMetadataAsMap() { +// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// // TODO omit attributes that don't have values +// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} +// return linkingMetadata; +// } +// +// public static String getLinkingMetadataAsString() { +// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// // TODO omit attributes that don't have values +// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} +// return linkingMetadata.toString(); +// } +} diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java deleted file mode 100644 index fae166c372..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_Instrumentation.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.nr.agent.instrumentation.log4j2; - -import com.newrelic.api.agent.NewRelic; -import com.newrelic.api.agent.weaver.Weave; -import com.newrelic.api.agent.weaver.Weaver; -import org.apache.logging.log4j.core.LogEvent; - -@Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig") -public class LoggerConfig_Instrumentation { - - protected void callAppenders(LogEvent event) { - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); - Weaver.callOriginal(); - } - -} \ No newline at end of file diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java new file mode 100644 index 0000000000..c2e530d7a2 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -0,0 +1,31 @@ +package org.apache.logging.log4j.core.config; + +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.log4j2.AgentUtil; +import org.apache.logging.log4j.core.LogEvent; + +@Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig", type = MatchType.ExactClass) +public class LoggerConfig_Instrumentation { + + protected void callAppenders(LogEvent event) { + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); + AgentUtil.reportNewRelicLogEvent(event); + + Weaver.callOriginal(); + } + + +// private void processLogEvent(final LogEvent event, final LoggerConfig.LoggerConfigPredicate predicate) { +// event.setIncludeLocation(isIncludeLocation()); +// if (predicate.allow(this)) { +// callAppenders(event); +// } +// logParent(event, predicate); +// } + + +} \ No newline at end of file diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java new file mode 100644 index 0000000000..9a911249b3 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java @@ -0,0 +1,29 @@ +//package org.apache.logging.log4j.core.impl; +// +//import com.newrelic.api.agent.weaver.MatchType; +//import com.newrelic.api.agent.weaver.Weave; +//import com.newrelic.api.agent.weaver.WeaveAllConstructors; +//import org.apache.logging.log4j.core.LogEvent; +//import org.apache.logging.log4j.message.Message; +// +//@Weave(originalName = "org.apache.logging.log4j.core.impl.Log4jLogEvent", type = MatchType.ExactClass) +//public class Log4jLogEvent { //implements LogEvent +// +// private Message message; +// +// @WeaveAllConstructors +// Log4jLogEvent() { +//// message = message.; +// } +// +// +// // might need to copy NewRelicContextDataProvider to instrumentation and find a way to activate it +// // https://github.com/newrelic/java-log-extensions/blob/main/log4j2/src/main/java/com/newrelic/logging/log4j2/NewRelicContextDataProvider.java +// +// +// final LogEvent event = Log4jLogEvent.newBuilder() +// +// // checkout org.apache.logging.log4j.core.layout.PatternLayout.encode +// // org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText +// +//} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java new file mode 100644 index 0000000000..2051c98ce6 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java @@ -0,0 +1,8 @@ +//package org.apache.logging.log4j.message; +// +//import com.newrelic.api.agent.weaver.MatchType; +//import com.newrelic.api.agent.weaver.Weave; +// +//@Weave(originalName = "org.apache.logging.log4j.message.Message", type = MatchType.Interface) +//public interface Message { +//} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java new file mode 100644 index 0000000000..cbee943024 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java @@ -0,0 +1,5 @@ +//package org.apache.logging.log4j.spi; +// +//public abstract class AbstractLogger { +// +//} diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index ab0f0780cf..137755b136 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -15,7 +15,7 @@ import org.junit.runner.RunWith; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "com.nr.agent.instrumentation.log4j2" }) +@InstrumentationTestConfig(includePrefixes = { "org.apache.logging.log4j.core" }) public class LoggerConfig_InstrumentationTest extends TestCase { private static final String CAPTURED = "This log message should be captured"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 62b3fd511e..61e1cca64a 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -368,11 +368,12 @@ private void sendLogEventsForReserv return; } InitialSizedJsonArray params = new InitialSizedJsonArray(3); - params.add(runId); + params.add(runId); // TODO is runID needed for LogEvents? JSONObject metadata = new JSONObject(); metadata.put("reservoir_size", reservoirSize); metadata.put("events_seen", eventsSeen); + metadata.put("plugin.type", "nr-java-agent"); // TODO what should this be exactly? // TODO if this is used for log_event_data a conditional check will be needed for other attributes specific to log sender events params.add(metadata); From 58e8288e4282584947e6981f4b218b97aeb3b6ca Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 10:44:54 -0800 Subject: [PATCH 12/96] Update to use MELT format for LogEvent data --- .../com/newrelic/agent/model/LogEvent.java | 15 ++++------ .../agent/attributes/AttributeValidator.java | 7 +++++ .../newrelic/agent/config/ConfigConstant.java | 1 + .../service/logging/LogSenderServiceImpl.java | 14 ++++----- .../agent/transport/DataSenderImpl.java | 30 ++++++++++++------- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java index e5dc133608..208228f926 100644 --- a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -7,13 +7,11 @@ package com.newrelic.agent.model; -import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONStreamAware; import java.io.IOException; import java.io.Writer; -import java.util.Arrays; import java.util.Map; public class LogEvent extends AnalyticsEvent implements JSONStreamAware { @@ -25,22 +23,19 @@ public LogEvent(String type, long timestamp, Map attributes, flo this.mutablePriority = priority; } - public void setPriority(float priority) { - this.mutablePriority = priority; - } - @Override public float getPriority() { return mutablePriority; } + public void setPriority(float priority) { + this.mutablePriority = priority; + } + @SuppressWarnings("unchecked") @Override public void writeJSONString(Writer out) throws IOException { - JSONObject intrinsics = new JSONObject(); - intrinsics.put("type", getType()); -// intrinsics.put("timestamp", getTimestamp()); // this probably isn't needed as there is a timestamp added for the log event already - JSONArray.writeJSONString(Arrays.asList(intrinsics, getMutableUserAttributes()), out); + JSONObject.writeJSONString(getMutableUserAttributes(), out); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java index 13d76d4518..bd7e2fb3fb 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java @@ -12,6 +12,7 @@ import com.newrelic.agent.Agent; import com.newrelic.agent.Transaction; import com.newrelic.agent.config.ConfigConstant; +import com.newrelic.agent.service.logging.LogSenderServiceImpl; import java.math.BigDecimal; import java.math.BigInteger; @@ -161,6 +162,12 @@ private boolean validateAndLogKeyLength(String key, String methodCalled) { } private String truncateValue(String key, String value, String methodCalled) { + // TODO if methodCalled is from log sender then don't truncate attribute values + // or instead set a different MAX_USER_ATTRIBUTE_SIZE if we decide to impose a limit + // Maybe create an AttributeValidator interface and a LogEventAttributeValidator with it's own truncateValue implementation instead of doing this + if (methodCalled.equals(LogSenderServiceImpl.METHOD)) { + return value; + } String truncatedVal = truncateString(value, ConfigConstant.MAX_USER_ATTRIBUTE_SIZE); if (!value.equals(truncatedVal)) { Agent.LOG.log(Level.FINER, diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java index 771c328bc3..0bc7c63c77 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java @@ -10,4 +10,5 @@ public class ConfigConstant { public static final int MAX_USER_ATTRIBUTES = 64; public static final int MAX_USER_ATTRIBUTE_SIZE = 255; + public static final int MAX_LOG_EVENT_ATTRIBUTE_SIZE = 4096; // TODO what if any limit should be imposed on log event attributes? } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 3d32ac6f63..cc99631599 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -62,6 +62,8 @@ public class LogSenderServiceImpl extends AbstractService implements LogSenderSe private static final LoadingCache stringCache = Caffeine.newBuilder().maximumSize(1000) .expireAfterAccess(70, TimeUnit.SECONDS).executor(Runnable::run).build(key -> key); + public static final String METHOD = "add log sender event attribute"; + // TODO it's not clear that log sender events should be tied to transactions in any way protected final ExtendedTransactionListener transactionListener = new ExtendedTransactionListener() { @@ -246,7 +248,7 @@ private void storeEvent(String appName, String eventType, Map attribu private boolean logSenderEventsDisabled(String eventType) { if (!enabled) { - // TODO just ignore high security for now + // TODO just ignore high security for now? if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); } else { @@ -386,9 +388,7 @@ private static LogEvent createValidatedEvent(String eventType, Map at // possibly, the values. But there's an interaction: if the key or value is chopped // within the attribute sender, the modified value won't be "interned" in our map. - // FIXME probably don't need this AttributeSender for LogSenderEvents AttributeSender sender = new LogSenderEventAttributeSender(userAttributes); - final String method = "add log sender event attribute"; for (Map.Entry entry : attributes.entrySet()) { String key = (String) entry.getKey(); @@ -404,14 +404,14 @@ private static LogEvent createValidatedEvent(String eventType, Map at mapInternString(key); if (value instanceof String) { - sender.addAttribute(key, mapInternString((String) value), method); + sender.addAttribute(key, mapInternString((String) value), METHOD); } else if (value instanceof Number) { - sender.addAttribute(key, (Number) value, method); + sender.addAttribute(key, (Number) value, METHOD); } else if (value instanceof Boolean) { - sender.addAttribute(key, (Boolean) value, method); + sender.addAttribute(key, (Boolean) value, METHOD); } else { // Java Agent specific - toString the value. This allows for e.g. enums as arguments. - sender.addAttribute(key, mapInternString(value.toString()), method); + sender.addAttribute(key, mapInternString(value.toString()), METHOD); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 61e1cca64a..8384fe5d4f 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -361,24 +361,34 @@ private void sendAnalyticEventsForR invokeRunId(method, encoding, runId, params); } + // Sends LogEvent data in the MELT format for logs + // https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#log-attribute-example private void sendLogEventsForReservoir(String method, String encoding, int reservoirSize, int eventsSeen, Collection events) throws Exception { Object runId = agentRunId; if (runId == NO_AGENT_RUN_ID || events.isEmpty()) { return; } - InitialSizedJsonArray params = new InitialSizedJsonArray(3); - params.add(runId); // TODO is runID needed for LogEvents? - JSONObject metadata = new JSONObject(); - metadata.put("reservoir_size", reservoirSize); - metadata.put("events_seen", eventsSeen); - metadata.put("plugin.type", "nr-java-agent"); // TODO what should this be exactly? - // TODO if this is used for log_event_data a conditional check will be needed for other attributes specific to log sender events - params.add(metadata); + JSONObject commonAttributes = new JSONObject(); + commonAttributes.put("plugin.type", "nr-java-agent"); - // TODO reshape the format of the events to better fit log data - params.add(events); + // build attributes object + JSONObject attributes = new JSONObject(); + attributes.put("attributes", commonAttributes); + + // build common object + JSONObject common = new JSONObject(); + common.put("common", attributes); + + // build logs object + JSONObject logs = new JSONObject(); + logs.put("logs", events); + + // params is top level + InitialSizedJsonArray params = new InitialSizedJsonArray(3); + params.add(common); + params.add(logs); invokeRunId(method, encoding, runId, params); } From 01647c992acffd577e065e809183ead3a2f86484 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 13:25:27 -0800 Subject: [PATCH 13/96] Some cleanup --- .../classic/spi/LoggingEvent_Instrumentation.java | 9 --------- .../com/newrelic/agent/config/LogSenderConfigImpl.java | 7 ++++--- .../newrelic/agent/service/logging/LogSenderService.java | 1 - .../agent/service/logging/LogSenderServiceImpl.java | 5 +++-- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 19e88fb644..eb0adcbd80 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -72,15 +72,6 @@ public LoggingEvent_Instrumentation() { } public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { -// "message": "...", -// "timestamp": 1641579045527, -// "thread.name": "http-nio-8080-exec-7", -// "log.level": "ERROR", -// "logger.name": "org.springframework.samples.petclinic.system.CrashController", -// "class.name": "org.springframework.samples.petclinic.system.CrashController", -// "method.name": "triggerException", -// "line.number": 41, - HashMap logEventMap = new HashMap<>(getLinkingMetadataAsMap()); logEventMap.put("message", message); logEventMap.put("timeStamp", timeStamp); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java index cf12df56a3..3bbf32a5d9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java @@ -14,7 +14,7 @@ public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { public static final String MAX_SAMPLES_STORED_PROP = "max_samples_stored"; public static final int DEFAULT_MAX_SAMPLES_STORED = 10000; public static final String ENABLED_PROP = "enabled"; - public static final boolean DEFAULT_ENABLED = true; + public static final boolean DEFAULT_ENABLED = true; // TODO make off by default, add yaml config public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config.log_sending."; public static final String ENABLED = SYSTEM_PROPERTY_ROOT + ENABLED_PROP; // public static final String COLLECT_CUSTOM_EVENTS = "collect_custom_events"; @@ -25,7 +25,9 @@ public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { public LogSenderConfigImpl(Map pProps, boolean highSecurity) { super(pProps, SYSTEM_PROPERTY_ROOT); maxSamplesStored = getProperty(MAX_SAMPLES_STORED_PROP, DEFAULT_MAX_SAMPLES_STORED); - isEnabled = !highSecurity && initEnabled(); + // TODO ignores highSecurity for the time being. Should we respect it? +// isEnabled = !highSecurity && initEnabled(); + isEnabled = initEnabled(); } public boolean initEnabled() { @@ -40,7 +42,6 @@ public boolean initEnabled() { return storedMoreThan0 && configEnabled; } - // TODO ignore highSecurity for the time being static LogSenderConfigImpl createLogSenderConfig(Map settings, boolean highSecurity) { if (settings == null) { settings = Collections.emptyMap(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java index 826ca0bfae..cfbafd3973 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java @@ -22,7 +22,6 @@ public interface LogSenderService extends EventService, Insights { /** * Returns an insights instance used to track events created during a transaction. The events will be reported to * the Transaction's application, or to the default application if not in a transaction. - * */ Insights getTransactionInsights(AgentConfig config); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index cc99631599..62170e5e73 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -474,11 +474,12 @@ public void recordCustomEvent(String eventType, Map attributes) { ServiceFactory.getServiceManager().getLogSenderService().storeEvent(applicationName, event); } } else { - Agent.LOG.log(Level.WARNING, "LogSender event with invalid type of {0} was reported for a transaction but ignored." - + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); + Agent.LOG.log(Level.WARNING, "Event with invalid type of {0} was reported for a transaction but ignored." + + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); // TODO figure out limit "less than 256 chars" } } + @VisibleForTesting public List getEventsForTesting() { return new ArrayList<>(events); } From 6d55ba90f975ef6b503ec050da41f215c89882f4 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 17:01:05 -0800 Subject: [PATCH 14/96] Refactor to use AgentBridge. Some cleanup. --- .../java/com/newrelic/agent/bridge/Agent.java | 8 + .../com/newrelic/agent/bridge/NoOpAgent.java | 5 +- .../com/newrelic/agent/bridge/NoOpLogs.java | 24 ++ .../java/com/newrelic/api/agent/Logs.java | 26 ++ .../com/newrelic/agent/model/LogEvent.java | 7 +- .../IntrospectorLogSenderService.java | 11 +- .../instrumentation/log4j2/AgentUtil.java | 3 +- .../spi/LoggingEvent_Instrumentation.java | 3 +- .../logbackclassic12/AgentUtil.java | 28 ++ .../java/com/newrelic/agent/AgentImpl.java | 3 +- .../com/newrelic/agent/DummyTransaction.java | 13 +- .../java/com/newrelic/agent/MetricNames.java | 8 + .../java/com/newrelic/agent/Transaction.java | 9 +- .../com/newrelic/agent/TransactionData.java | 3 +- .../agent/attributes/AttributeSender.java | 4 +- .../service/logging/LogSenderService.java | 7 +- .../service/logging/LogSenderServiceImpl.java | 269 +++++++++++------- .../java/com/newrelic/api/agent/Agent.java | 9 - .../com/newrelic/api/agent/NoOpAgent.java | 5 - 19 files changed, 304 insertions(+), 141 deletions(-) create mode 100644 agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java create mode 100644 agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/Agent.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/Agent.java index b3af214510..acab69947a 100644 --- a/agent-bridge/src/main/java/com/newrelic/agent/bridge/Agent.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/Agent.java @@ -7,6 +7,7 @@ package com.newrelic.agent.bridge; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.NewRelic; /** @@ -87,4 +88,11 @@ public interface Agent extends com.newrelic.api.agent.Agent { */ boolean ignoreIfUnstartedAsyncContext(Object activityContext); + /** + * Provides access to the LogSender events API. + * + * @return Object used to add custom events. + */ + Logs getLogSender(); + } diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java index 4e53dff0fc..68b610559d 100644 --- a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpAgent.java @@ -10,6 +10,7 @@ import com.newrelic.api.agent.Config; import com.newrelic.api.agent.Insights; import com.newrelic.api.agent.Logger; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.MetricAggregator; import com.newrelic.api.agent.TraceMetadata; @@ -64,8 +65,8 @@ public Insights getInsights() { } @Override - public Insights getLogSender() { - return NoOpInsights.INSTANCE; + public Logs getLogSender() { + return NoOpLogs.INSTANCE; } @Override diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java new file mode 100644 index 0000000000..9feeadb058 --- /dev/null +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java @@ -0,0 +1,24 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.bridge; + +import com.newrelic.api.agent.Logs; + +import java.util.Map; + +class NoOpLogs implements Logs { + static final Logs INSTANCE = new NoOpLogs(); + + private NoOpLogs() { + } + + @Override + public void recordLogEvent(Map attributes) { + } + +} diff --git a/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java new file mode 100644 index 0000000000..f0b262fb4f --- /dev/null +++ b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java @@ -0,0 +1,26 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.api.agent; + +import java.util.Map; + +/** + * Used to send LogEvents to New Relic. Each LogEvent represents a single log line. + */ +public interface Logs { + + /** + * Sends a LogEvent for the current application. + * + * @param attributes A map of log event data (e.g. log message, log timestamp, log level) + * Each key should be a String and each value should be a String, Number, or Boolean. + * For map values that are not String, Number, or Boolean object types the toString value will be used. + * @since 7.6.0 + */ + void recordLogEvent(Map attributes); +} diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java index 208228f926..bb1f9604f1 100644 --- a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -16,10 +16,13 @@ public class LogEvent extends AnalyticsEvent implements JSONStreamAware { + public static final String LOG_EVENT_TYPE = "LogEvent"; + private volatile float mutablePriority; - public LogEvent(String type, long timestamp, Map attributes, float priority) { - super(type, timestamp, priority, attributes); + // FIXME probably don't need to pass timestamp as we use the value from the log event captured in the library instrumentation + public LogEvent(long timestamp, Map attributes, float priority) { + super(LOG_EVENT_TYPE, timestamp, priority, attributes); this.mutablePriority = priority; } diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java index 39ff5a7da5..bdc25ef113 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -20,6 +20,7 @@ import com.newrelic.agent.service.logging.LogSenderService; import com.newrelic.agent.tracing.DistributedTraceServiceImpl; import com.newrelic.api.agent.Insights; +import com.newrelic.api.agent.Logs; import java.util.ArrayList; import java.util.Collection; @@ -27,6 +28,8 @@ import java.util.List; import java.util.Map; +import static com.newrelic.agent.model.LogEvent.LOG_EVENT_TYPE; + class IntrospectorLogSenderService implements LogSenderService { private static String SERVICE_NAME = "LogSenderService"; @@ -76,16 +79,16 @@ public boolean isStoppedOrStopping() { } @Override - public void recordCustomEvent(String eventType, Map attributes) { - if (AnalyticsEvent.isValidType(eventType)) { + public void recordLogEvent(Map attributes) { + if (AnalyticsEvent.isValidType(LOG_EVENT_TYPE)) { Map atts = Maps.newHashMap(attributes); - LogEvent event = new LogEvent(eventType, System.currentTimeMillis(), atts, DistributedTraceServiceImpl.nextTruncatedFloat()); + LogEvent event = new LogEvent(System.currentTimeMillis(), atts, DistributedTraceServiceImpl.nextTruncatedFloat()); storeEvent("TestApp", event); } } @Override - public Insights getTransactionInsights(AgentConfig config) { + public Logs getTransactionLogs(AgentConfig config) { return this; } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index da81e0f2fc..bfeb4cedc9 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -1,5 +1,6 @@ package com.nr.agent.instrumentation.log4j2; +import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; @@ -33,7 +34,7 @@ public static void reportNewRelicLogEvent(LogEvent event) { Throwable throwable = event.getThrown(); logEventMap.put("throwable", throwable != null ? throwable.toString() : EMPTY_STRING); - NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index eb0adcbd80..5cbb3d6693 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -3,6 +3,7 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; +import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; @@ -106,7 +107,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str timeStamp = System.currentTimeMillis(); - NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 2f77f4bc78..836ba8a90a 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -2,10 +2,38 @@ import com.newrelic.api.agent.NewRelic; +import java.util.HashMap; import java.util.Map; public class AgentUtil { +// public static void reportNewRelicLogEvent(LogEvent event) { +// final String EMPTY_STRING = ""; +// +// if (event != null) { +// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); +// +// Message message = event.getMessage(); +// logEventMap.put("message", message != null ? message.toString() : EMPTY_STRING); +// logEventMap.put("timeStampTimeMillis", event.getTimeMillis()); +// +// Instant instant = event.getInstant(); +// logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); +// +// Level level = event.getLevel(); +// logEventMap.put("log.level", level != null ? level.toString() : EMPTY_STRING); +// logEventMap.put("logger.name", event.getLoggerName()); +// logEventMap.put("class.name", event.getLoggerFqcn()); +// +// Throwable throwable = event.getThrown(); +// logEventMap.put("throwable", throwable != null ? throwable.toString() : EMPTY_STRING); +// +// NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); +// } +// } + + public static Map getLinkingMetadataAsMap() { Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); // TODO omit attributes that don't have values diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java index ea554d22f3..a804458839 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java @@ -18,6 +18,7 @@ import com.newrelic.agent.tracers.Tracer; import com.newrelic.api.agent.Insights; import com.newrelic.api.agent.Logger; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.MetricAggregator; import com.newrelic.api.agent.TraceMetadata; @@ -132,7 +133,7 @@ public Insights getInsights() { } @Override - public Insights getLogSender() { + public Logs getLogSender() { return ServiceFactory.getServiceManager().getLogSenderService(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java index db9a21e644..10ba4863c6 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java @@ -37,10 +37,8 @@ import com.newrelic.agent.tracers.ClassMethodSignature; import com.newrelic.agent.tracers.MetricNameFormatWithHost; import com.newrelic.agent.tracers.NoOpTracer; -import com.newrelic.agent.tracers.OtherRootTracer; import com.newrelic.agent.tracers.Tracer; import com.newrelic.agent.tracers.TracerFactory; -import com.newrelic.agent.tracers.metricname.SimpleMetricNameFormat; import com.newrelic.agent.transaction.PriorityTransactionName; import com.newrelic.agent.transaction.TransactionCache; import com.newrelic.agent.transaction.TransactionCounts; @@ -50,6 +48,7 @@ import com.newrelic.api.agent.ApplicationNamePriority; import com.newrelic.api.agent.InboundHeaders; import com.newrelic.api.agent.Insights; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.MetricAggregator; import com.newrelic.api.agent.OutboundHeaders; import com.newrelic.api.agent.Request; @@ -79,9 +78,9 @@ public class DummyTransaction extends Transaction { private final Object lock = new Object(); private final Insights insights = new DummyInsights(); - private final Insights logEvents = new DummyLogEvents(); + private final Logs logEvents = new DummyLogEvents(); private final AgentConfig defaultConfig; - private final TracerList tracerList = new TracerList(null, new DummySet()); + private final TracerList tracerList = new TracerList(null, new DummySet<>()); private final TransactionTimer timer = new TransactionTimer(0); private final InboundHeaderState inboundHeaderState = new InboundHeaderState(null, null); private final SlowQueryListener slowQueryListener = new NopSlowQueryListener(); @@ -172,7 +171,7 @@ public Insights getInsightsData() { } @Override - public Insights getLogEventData() { + public Logs getLogEventData() { return logEvents; } @@ -657,9 +656,9 @@ public void recordCustomEvent(String eventType, Map attributes) { } } - static final class DummyLogEvents implements Insights { + static final class DummyLogEvents implements Logs { @Override - public void recordCustomEvent(String eventType, Map attributes) { + public void recordLogEvent(Map attributes) { } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java index 051b6aa6d7..32bd3a2772 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java @@ -218,14 +218,19 @@ public class MetricNames { public static final String SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SENT = "Supportability/Events/Customer/Sent"; public static final String SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SEEN = "Supportability/Events/Customer/Seen"; + public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SENT = "Supportability/LogEvents/Customer/Sent"; + public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SEEN = "Supportability/LogEvents/Customer/Seen"; + public static final String SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/ReportPeriod"; public static final String SUPPORTABILITY_ERROR_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/ErrorEventData/ReportPeriod"; public static final String SUPPORTABILITY_INSIGHTS_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/CustomEventData/ReportPeriod"; + public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/LogEventData/ReportPeriod"; public static final String SUPPORTABILITY_SPAN_EVENT_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/SpanEventData/ReportPeriod"; public static final String SUPPORTABILITY_ANALYTIC_EVENT_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/AnalyticEventData/ReportPeriod"; public static final String SUPPORTABILITY_ERROR_EVENT_DATA_HARVEST_LIMIT = "Supportability/EventHarvest/ErrorEventData/HarvestLimit"; public static final String SUPPORTABILITY_CUSTOM_EVENT_DATA_HARVEST_LIMIT = "Supportability/EventHarvest/CustomEventData/HarvestLimit"; + public static final String SUPPORTABILITY_LOG_EVENT_DATA_HARVEST_LIMIT = "Supportability/EventHarvest/LogEventData/HarvestLimit"; public static final String SUPPORTABILITY_ANALYTIC_EVENT_DATA_HARVEST_LIMIT = "Supportability/EventHarvest/AnalyticEventData/HarvestLimit"; public static final String SUPPORTABILITY_SPAN_EVENT_DATA_HARVEST_LIMIT = "Supportability/EventHarvest/SpanEventData/HarvestLimit"; @@ -245,6 +250,9 @@ public class MetricNames { public static final String SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_INTERVAL = "Supportability/EventHarvest/Customer/interval"; public static final String SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_TRANSMIT = "Supportability/EventHarvest/Customer/transmit"; + public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_EVENT_HARVEST_INTERVAL = "Supportability/EventHarvest/LogEvent/interval"; + public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_EVENT_HARVEST_TRANSMIT = "Supportability/EventHarvest/LogEvent/transmit"; + public static final String SUPPORTABILITY_SPAN_SERVICE_EVENT_HARVEST_INTERVAL = "Supportability/EventHarvest/SpanEvent/interval"; public static final String SUPPORTABILITY_SPAN_SERVICE_EVENT_HARVEST_TRANSMIT = "Supportability/EventHarvest/SpanEvent/transmit"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java index f1ae36373a..584f1e9f75 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java @@ -22,6 +22,7 @@ import com.newrelic.agent.bridge.NoOpToken; import com.newrelic.agent.bridge.Token; import com.newrelic.agent.bridge.TransactionNamePriority; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.TransportType; import com.newrelic.agent.bridge.WebResponse; import com.newrelic.agent.browser.BrowserTransactionState; @@ -192,7 +193,7 @@ public class Transaction { private final AtomicReference insights; // Insights events added by the user during this transaction - private final AtomicReference logEvents; + private final AtomicReference logEvents; // contains all work currently running private final Map runningChildren; @@ -617,11 +618,11 @@ public Insights getInsightsData() { return insightsData; } - public Insights getLogEventData() { - Insights logEventData = logEvents.get(); + public Logs getLogEventData() { + Logs logEventData = logEvents.get(); if (logEventData == null) { AgentConfig defaultConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); - logEvents.compareAndSet(null, ServiceFactory.getServiceManager().getLogSenderService().getTransactionInsights(defaultConfig)); + logEvents.compareAndSet(null, ServiceFactory.getServiceManager().getLogSenderService().getTransactionLogs(defaultConfig)); logEventData = logEvents.get(); } return logEventData; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java b/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java index 940809103f..b5373ac1d0 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/TransactionData.java @@ -8,6 +8,7 @@ package com.newrelic.agent; import com.newrelic.agent.attributes.AttributesService; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.TransportType; import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.config.AgentConfigImpl; @@ -48,7 +49,7 @@ public Insights getInsightsData() { return tx.getInsightsData(); } - public Insights getLogEventData() { + public Logs getLogEventData() { return tx.getLogEventData(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeSender.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeSender.java index f7da2a95b7..50eaffc28d 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeSender.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeSender.java @@ -21,9 +21,9 @@ public AttributeSender(AttributeValidator attributeValidator) { } /** - * This is used only for logging. + * This is used only for agent logging. * - * @return The type of attribute (e.g. "agent", "custom") + * @return The type of attribute (e.g. "agent", "custom", "log") */ protected abstract String getAttributeType(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java index cfbafd3973..34e9e32754 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java @@ -11,19 +11,20 @@ import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.service.EventService; import com.newrelic.api.agent.Insights; +import com.newrelic.api.agent.Logs; /** * LogSenderService interface * - * Extending Insights makes the recordCustomEvent(...) API available to implementing classes + * Extending Logs makes the recordLogEvent(...) API available to implementing classes */ -public interface LogSenderService extends EventService, Insights { +public interface LogSenderService extends EventService, Logs { /** * Returns an insights instance used to track events created during a transaction. The events will be reported to * the Transaction's application, or to the default application if not in a transaction. */ - Insights getTransactionInsights(AgentConfig config); + Logs getTransactionLogs(AgentConfig config); /** * Store event into Reservoir following usual sampling using the given appName. Preference should be given to diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 62170e5e73..160a61f9b5 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -20,7 +20,6 @@ import com.newrelic.agent.attributes.AttributeValidator; import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.config.AgentConfigListener; -import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.service.AbstractService; import com.newrelic.agent.service.ServiceFactory; @@ -30,7 +29,7 @@ import com.newrelic.agent.stats.TransactionStats; import com.newrelic.agent.tracing.DistributedTraceServiceImpl; import com.newrelic.agent.transport.HttpError; -import com.newrelic.api.agent.Insights; +import com.newrelic.api.agent.Logs; import java.text.MessageFormat; import java.util.ArrayList; @@ -45,28 +44,31 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; +import static com.newrelic.agent.model.LogEvent.LOG_EVENT_TYPE; + public class LogSenderServiceImpl extends AbstractService implements LogSenderService { - // Whether the service as a whole is enabled. Disabling shuts down all analytic events for transactions. + // Whether the service as a whole is enabled. Disabling shuts down all log events for transactions. private volatile boolean enabled; // Key is the app name, value is if it is enabled - should be a limited number of names private final ConcurrentMap isEnabledForApp = new ConcurrentHashMap<>(); /* - * Number of events in the reservoir sampling buffer per-app. All apps get the same value. Synthetics are buffered - * separately per app using a deterministic algorithm. + * Number of log events in the reservoir sampling buffer per-app. All apps get the same value. */ private volatile int maxSamplesStored; - // Key is app name, value is collection of per-transaction analytic events for next harvest for that app. + // Key is app name, value is collection of per-transaction log events for next harvest for that app. private final ConcurrentHashMap> reservoirForApp = new ConcurrentHashMap<>(); private static final LoadingCache stringCache = Caffeine.newBuilder().maximumSize(1000) .expireAfterAccess(70, TimeUnit.SECONDS).executor(Runnable::run).build(key -> key); - public static final String METHOD = "add log sender event attribute"; + public static final String METHOD = "add log event attribute"; + public static final String LOG_SENDER_SERVICE = "Log Sender Service"; - // TODO it's not clear that log sender events should be tied to transactions in any way + /** + * Lifecycle listener for log events associated with a transaction + */ protected final ExtendedTransactionListener transactionListener = new ExtendedTransactionListener() { - @Override public void dispatcherTransactionStarted(Transaction transaction) { } @@ -74,20 +76,22 @@ public void dispatcherTransactionStarted(Transaction transaction) { @Override public void dispatcherTransactionFinished(TransactionData transactionData, TransactionStats transactionStats) { // FIXME not sure this is a great idea to store log events for the duration of a transaction... - TransactionInsights data = (TransactionInsights) transactionData.getLogEventData(); + TransactionLogs data = (TransactionLogs) transactionData.getLogEventData(); storeEvents(transactionData.getApplicationName(), transactionData.getPriority(), data.events); } @Override public void dispatcherTransactionCancelled(Transaction transaction) { // FIXME not sure this is a great idea to store log events for the duration of a transaction... - // Even if the transaction is cancelled we still want to send up any events that were held in it - TransactionInsights data = (TransactionInsights) transaction.getLogEventData(); + // Even if the transaction is canceled we still want to send up any events that were held in it + TransactionLogs data = (TransactionLogs) transaction.getLogEventData(); storeEvents(transaction.getApplicationName(), transaction.getPriority(), data.events); } - }; + /** + * Listener to detect changes to the agent config + */ protected final AgentConfigListener configListener = new AgentConfigListener() { @Override public void configChanged(String appName, AgentConfig agentConfig) { @@ -97,7 +101,7 @@ public void configChanged(String appName, AgentConfig agentConfig) { } }; - private List harvestables = new ArrayList<>(); + private final List harvestables = new ArrayList<>(); public LogSenderServiceImpl() { super(LogSenderServiceImpl.class.getSimpleName()); @@ -107,11 +111,19 @@ public LogSenderServiceImpl() { isEnabledForApp.put(config.getApplicationName(), enabled); } + /** + * Whether the LogSenderService is enabled or not + * @return true if enabled, else false + */ @Override public boolean isEnabled() { return enabled; } + /** + * Start the LogSenderService + * @throws Exception if service fails to start + */ @Override protected void doStart() throws Exception { // TODO it's not clear that log sender events should be tied to transactions in any way @@ -119,6 +131,10 @@ protected void doStart() throws Exception { ServiceFactory.getConfigService().addIAgentConfigListener(configListener); } + /** + * Stop the LogSenderService + * @throws Exception if service fails to stop + */ @Override protected void doStop() throws Exception { removeHarvestables(); @@ -136,49 +152,66 @@ private void removeHarvestables() { } } + /** + * Records a LogEvent. If a LogEvent occurs within a Transaction it will be associated with it. + * @param attributes A map of log event data (e.g. log message, log timestamp, log level) + * Each key should be a String and each value should be a String, Number, or Boolean. + * For map values that are not String, Number, or Boolean object types the toString value will be used. + */ @Override - public void recordCustomEvent(String eventType, Map attributes) { - if (logSenderEventsDisabled(eventType)) { + public void recordLogEvent(Map attributes) { + if (logEventsDisabled()) { return; } - if (AnalyticsEvent.isValidType(eventType)) { - Transaction transaction = ServiceFactory.getTransactionService().getTransaction(false); - // FIXME perhaps ignore transaction status and just always send log events... - // what is the benefit of storing them on the transaction? Sampling maybe? - if (transaction == null || !transaction.isInProgress() || transaction.isIgnore()) { - String applicationName = ServiceFactory.getRPMService().getApplicationName(); - if (transaction != null && transaction.getApplicationName() != null) { - applicationName = transaction.getApplicationName(); - } - AgentConfig agentConfig = ServiceFactory.getConfigService().getAgentConfig(applicationName); - if (!getIsEnabledForApp(agentConfig, applicationName)) { - reservoirForApp.remove(applicationName); - return; - } - storeEvent(applicationName, eventType, attributes); - } else { - // FIXME not sure this is a great idea to store log events for the duration of a transaction... - transaction.getLogEventData().recordCustomEvent(eventType, attributes); + Transaction transaction = ServiceFactory.getTransactionService().getTransaction(false); + // FIXME perhaps ignore transaction status and just always send log events... + // what is the benefit of storing them on the transaction? Sampling? + // Not in a Transaction or an existing Transaction is not in progress or is ignored + if (transaction == null || !transaction.isInProgress() || transaction.isIgnore()) { + String applicationName = ServiceFactory.getRPMService().getApplicationName(); + + if (transaction != null && transaction.getApplicationName() != null) { + applicationName = transaction.getApplicationName(); + } + + AgentConfig agentConfig = ServiceFactory.getConfigService().getAgentConfig(applicationName); + + if (!getIsEnabledForApp(agentConfig, applicationName)) { + reservoirForApp.remove(applicationName); + return; } - MetricNames.recordApiSupportabilityMetric(MetricNames.SUPPORTABILITY_API_RECORD_LOG_EVENT); + createAndStoreEvent(applicationName, attributes); + // In a Transaction that is in progress and not ignored } else { - Agent.LOG.log(Level.WARNING, "Custom event with invalid type of {0} was reported but ignored." - + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); + // FIXME not sure this is a great idea to store log events for the duration of a transaction... + transaction.getLogEventData().recordLogEvent(attributes); } + MetricNames.recordApiSupportabilityMetric(MetricNames.SUPPORTABILITY_API_RECORD_LOG_EVENT); } + /** + * Store a collection of LogEvents in the priority queue when a Transaction is finished or cancelled + * + * @param appName app name + * @param priority sampling priority from Transaction + * @param events collection of LogEvents to store + */ private void storeEvents(String appName, float priority, Collection events) { if (events.size() > 0) { DistributedSamplingPriorityQueue eventList = getReservoir(appName); for (LogEvent event : events) { - // Set "priority" on LogSenderEvent based on priority value from Transaction + // Set "priority" on LogEvent based on priority value from Transaction event.setPriority(priority); eventList.add(event); } } } + /** + * Register LogSenderHarvestable + * @param appName application name + */ public void addHarvestableToService(String appName) { Harvestable harvestable = new LogSenderHarvestableImpl(this, appName); ServiceFactory.getHarvestService().addHarvestable(harvestable); @@ -225,9 +258,14 @@ public void harvestPendingEvents() { } } + /** + * Store a LogEvent instance + * @param appName application name + * @param event log event + */ @Override public void storeEvent(String appName, LogEvent event) { - if (logSenderEventsDisabled(event.getType())) { + if (logEventsDisabled()) { return; } @@ -236,24 +274,36 @@ public void storeEvent(String appName, LogEvent event) { Agent.LOG.finest(MessageFormat.format("Added Custom Event of type {0}", event.getType())); } - private void storeEvent(String appName, String eventType, Map attributes) { - if (logSenderEventsDisabled(eventType)) { + /** + * Create and store a LogEvent instance + * @param appName application name + * @param attributes Map of attributes to create a LogEvent from + */ + private void createAndStoreEvent(String appName, Map attributes) { + if (logEventsDisabled()) { return; } DistributedSamplingPriorityQueue eventList = getReservoir(appName); - eventList.add(createValidatedEvent(eventType, attributes)); - Agent.LOG.finest(MessageFormat.format("Added Custom Event of type {0}", eventType)); + eventList.add(createValidatedEvent(attributes)); + Agent.LOG.finest(MessageFormat.format("Added event of type {0}", LOG_EVENT_TYPE)); } - private boolean logSenderEventsDisabled(String eventType) { + /** + * Check if LogEvents are disabled + * + * @return true if they are disabled, false if they are enabled + */ + private boolean logEventsDisabled() { if (!enabled) { - // TODO just ignore high security for now? - if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { - Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); - } else { - Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", eventType); - } + // TODO high security is disabled for now. How should we handle it? +// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { +// Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); +// } else { +// Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", eventType); +// } + + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); return true; // Log Sender events are disabled } @@ -261,17 +311,28 @@ private boolean logSenderEventsDisabled(String eventType) { return false; // Log Sender events are enabled } + /** + * Get the LogEvent reservoir + * + * @param appName app name + * @return Queue of LogEvent instances + */ @VisibleForTesting public DistributedSamplingPriorityQueue getReservoir(String appName) { DistributedSamplingPriorityQueue result = reservoirForApp.get(appName); while (result == null) { // I don't think this loop can actually execute more than once, but it's prudent to assume it can. - reservoirForApp.putIfAbsent(appName, new DistributedSamplingPriorityQueue(appName, "Log Sender Service", maxSamplesStored)); + reservoirForApp.putIfAbsent(appName, new DistributedSamplingPriorityQueue<>(appName, LOG_SENDER_SERVICE, maxSamplesStored)); result = reservoirForApp.get(appName); } return result; } + /** + * Harvest and send the LogEvents + * + * @param appName the application to harvest for + */ public void harvestEvents(final String appName) { if (!getIsEnabledForApp(ServiceFactory.getConfigService().getAgentConfig(appName), appName)) { reservoirForApp.remove(appName); @@ -285,15 +346,15 @@ public void harvestEvents(final String appName) { long startTimeInNanos = System.nanoTime(); final DistributedSamplingPriorityQueue reservoir = this.reservoirForApp.put(appName, - new DistributedSamplingPriorityQueue<>(appName, "Log Sender Service", maxSamplesStored)); + new DistributedSamplingPriorityQueue<>(appName, LOG_SENDER_SERVICE, maxSamplesStored)); if (reservoir != null && reservoir.size() > 0) { try { - // TODO actual sending of events - + // Send LogEvents ServiceFactory.getRPMServiceManager() .getOrCreateRPMService(appName) .sendLogEvents(maxSamplesStored, reservoir.getNumberOfTries(), Collections.unmodifiableList(reservoir.asList())); + final long durationInNanos = System.nanoTime() - startTimeInNanos; ServiceFactory.getStatsService().doStatsWork(new StatsWork() { @Override @@ -309,49 +370,49 @@ public String getAppName() { if (reservoir.size() < reservoir.getNumberOfTries()) { int dropped = reservoir.getNumberOfTries() - reservoir.size(); - Agent.LOG.log(Level.FINE, "Dropped {0} custom events out of {1}.", dropped, reservoir.getNumberOfTries()); + Agent.LOG.log(Level.FINE, "Dropped {0} log events out of {1}.", dropped, reservoir.getNumberOfTries()); } } catch (HttpError e) { if (!e.discardHarvestData()) { - Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be included in the next harvest.", e); + Agent.LOG.log(Level.FINE, "Unable to send log events. Unsent events will be included in the next harvest.", e); // Save unsent data by merging it with current data using reservoir algorithm DistributedSamplingPriorityQueue currentReservoir = reservoirForApp.get(appName); currentReservoir.retryAll(reservoir); } else { // discard harvest data reservoir.clear(); - Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be dropped.", e); + Agent.LOG.log(Level.FINE, "Unable to send log events. Unsent events will be dropped.", e); } } catch (Exception e) { // discard harvest data reservoir.clear(); - Agent.LOG.log(Level.FINE, "Unable to send custom events. Unsent events will be dropped.", e); + Agent.LOG.log(Level.FINE, "Unable to send log events. Unsent events will be dropped.", e); } } } @Override public String getEventHarvestIntervalMetric() { - return MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_INTERVAL; + return MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_EVENT_HARVEST_INTERVAL; } @Override public String getReportPeriodInSecondsMetric() { - return MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_REPORT_PERIOD_IN_SECONDS; + return MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_REPORT_PERIOD_IN_SECONDS; } @Override public String getEventHarvestLimitMetric() { - return MetricNames.SUPPORTABILITY_CUSTOM_EVENT_DATA_HARVEST_LIMIT; + return MetricNames.SUPPORTABILITY_LOG_EVENT_DATA_HARVEST_LIMIT; } private void recordSupportabilityMetrics(StatsEngine statsEngine, long durationInNanoseconds, DistributedSamplingPriorityQueue reservoir) { - statsEngine.getStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SENT) + statsEngine.getStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SENT) .incrementCallCount(reservoir.size()); - statsEngine.getStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SEEN) + statsEngine.getStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SEEN) .incrementCallCount(reservoir.getNumberOfTries()); - statsEngine.getResponseTimeStats(MetricNames.SUPPORTABILITY_INSIGHTS_SERVICE_EVENT_HARVEST_TRANSMIT) + statsEngine.getResponseTimeStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_EVENT_HARVEST_TRANSMIT) .recordResponseTime(durationInNanoseconds, TimeUnit.NANOSECONDS); } @@ -378,25 +439,31 @@ private static String mapInternString(String value) { return stringCache.get(value); } - private static LogEvent createValidatedEvent(String eventType, Map attributes) { + /** + * Create a validated LogEvent + * @param attributes Map of attributes to create a LogEvent from + * @return LogEvent instance + */ + private static LogEvent createValidatedEvent(Map attributes) { Map userAttributes = new HashMap<>(attributes.size()); - LogEvent event = new LogEvent(mapInternString(eventType), System.currentTimeMillis(), userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); + // FIXME LogEvent constructor only needs the timestamp for the AnalyticsEvent super class but it won't + // actually be added to the LogEvent as it isn't needed. We use the timestamp captured from the log library. + LogEvent event = new LogEvent(System.currentTimeMillis(), userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); // Now add the attributes from the argument map to the event using an AttributeSender. // An AttributeSender is the way to reuse all the existing attribute validations. We // also locally "intern" Strings because we anticipate a lot of reuse of the keys and, // possibly, the values. But there's an interaction: if the key or value is chopped // within the attribute sender, the modified value won't be "interned" in our map. + AttributeSender sender = new LogEventAttributeSender(userAttributes); - AttributeSender sender = new LogSenderEventAttributeSender(userAttributes); - - for (Map.Entry entry : attributes.entrySet()) { - String key = (String) entry.getKey(); + for (Map.Entry entry : attributes.entrySet()) { + String key = entry.getKey(); Object value = entry.getValue(); // key or value is null, skip it with a log message and iterate to next entry in attributes.entrySet() if (key == null || value == null) { - Agent.LOG.log(Level.WARNING, "Log Sender event with invalid attributes key or value of null was reported for a transaction but ignored." + Agent.LOG.log(Level.WARNING, "Log event with invalid attributes key or value of null was reported for a transaction but ignored." + " Each key should be a String and each value should be a String, Number, or Boolean."); continue; } @@ -418,13 +485,16 @@ private static LogEvent createValidatedEvent(String eventType, Map at return event; } - private static class LogSenderEventAttributeSender extends AttributeSender { + /** + * Validate attributes and add them to LogEvents + */ + private static class LogEventAttributeSender extends AttributeSender { - private static final String ATTRIBUTE_TYPE = "custom"; + private static final String ATTRIBUTE_TYPE = "log"; private final Map userAttributes; - public LogSenderEventAttributeSender(Map userAttributes) { + public LogEventAttributeSender(Map userAttributes) { super(new AttributeValidator(ATTRIBUTE_TYPE)); this.userAttributes = userAttributes; setTransactional(false); @@ -437,45 +507,46 @@ protected String getAttributeType() { @Override protected Map getAttributeMap() { - if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { - return userAttributes; - } - return null; + // FIXME skip this check for now as it isn't clear what to do with Log data if LASP or high security are enabled +// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { +// return userAttributes; +// } +// return null; + return userAttributes; } } @Override - public Insights getTransactionInsights(AgentConfig config) { - return new TransactionInsights(config); + public Logs getTransactionLogs(AgentConfig config) { + return new TransactionLogs(config); } - public static final class TransactionInsights implements Insights { + /** + * Used to record LogEvents on Transactions + */ + public static final class TransactionLogs implements Logs { final LinkedBlockingQueue events; - TransactionInsights(AgentConfig config) { + TransactionLogs(AgentConfig config) { int maxSamplesStored = config.getLogSenderConfig().getMaxSamplesStored(); events = new LinkedBlockingQueue<>(maxSamplesStored); } @Override - public void recordCustomEvent(String eventType, Map attributes) { - if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { - Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); - return; - } - - if (AnalyticsEvent.isValidType(eventType)) { - LogEvent event = createValidatedEvent(eventType, attributes); - if (events.offer(event)) { - Agent.LOG.finest(MessageFormat.format("Added event of type {0} in Transaction.", eventType)); - } else { - // Too many events are cached on the transaction, send directly to the reservoir. - String applicationName = ServiceFactory.getRPMService().getApplicationName(); - ServiceFactory.getServiceManager().getLogSenderService().storeEvent(applicationName, event); - } + public void recordLogEvent(Map attributes) { + // TODO ignore high security for now +// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { +// Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); +// return; +// } + + LogEvent event = createValidatedEvent(attributes); + if (events.offer(event)) { + Agent.LOG.finest(MessageFormat.format("Added event of type {0} in Transaction.", LOG_EVENT_TYPE)); } else { - Agent.LOG.log(Level.WARNING, "Event with invalid type of {0} was reported for a transaction but ignored." - + " Event types must match /^[a-zA-Z0-9:_ ]+$/, be non-null, and less than 256 chars.", eventType); // TODO figure out limit "less than 256 chars" + // Too many events are cached on the transaction, send directly to the reservoir. + String applicationName = ServiceFactory.getRPMService().getApplicationName(); + ServiceFactory.getServiceManager().getLogSenderService().storeEvent(applicationName, event); } } diff --git a/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java b/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java index 0a8f0ab9a7..b785f3f3a7 100644 --- a/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java +++ b/newrelic-api/src/main/java/com/newrelic/api/agent/Agent.java @@ -63,15 +63,6 @@ public interface Agent { */ Insights getInsights(); - /** - * Provides access to the LogSender custom events API. - * - * @return Object used to add custom events. - * @since 3.13.0 - */ - // FIXME this should probably be on the agent bridge instead of a public API - Insights getLogSender(); - /** * Provides access to the Trace Metadata API for details about the currently executing distributed trace. * diff --git a/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java b/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java index ed1d22a3b5..8f26efc2f7 100644 --- a/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java +++ b/newrelic-api/src/main/java/com/newrelic/api/agent/NoOpAgent.java @@ -439,11 +439,6 @@ public Insights getInsights() { return INSIGHTS; } - @Override - public Insights getLogSender() { - return INSIGHTS; - } - @Override public TraceMetadata getTraceMetadata() { return TRACE_METADATA; From 6a0d0c5404efde95aa28232a8a1c985b028952e4 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 17:41:06 -0800 Subject: [PATCH 15/96] Cleanup instrumentation --- .../instrumentation/log4j2/AgentUtil.java | 21 -------------- .../logbackclassic12/AgentUtil.java | 28 ------------------- 2 files changed, 49 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index bfeb4cedc9..93dcbd000d 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -37,25 +37,4 @@ public static void reportNewRelicLogEvent(LogEvent event) { AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } } - -// public static Map createLogEventAttributesMap(LogEvent event) { -// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// // TODO omit attributes that don't have values -// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} -// return linkingMetadata; -// } -// -// public static Map getLinkingMetadataAsMap() { -// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// // TODO omit attributes that don't have values -// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} -// return linkingMetadata; -// } -// -// public static String getLinkingMetadataAsString() { -// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// // TODO omit attributes that don't have values -// // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} -// return linkingMetadata.toString(); -// } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 836ba8a90a..2f77f4bc78 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -2,38 +2,10 @@ import com.newrelic.api.agent.NewRelic; -import java.util.HashMap; import java.util.Map; public class AgentUtil { -// public static void reportNewRelicLogEvent(LogEvent event) { -// final String EMPTY_STRING = ""; -// -// if (event != null) { -// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); -// -// Message message = event.getMessage(); -// logEventMap.put("message", message != null ? message.toString() : EMPTY_STRING); -// logEventMap.put("timeStampTimeMillis", event.getTimeMillis()); -// -// Instant instant = event.getInstant(); -// logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); -// -// Level level = event.getLevel(); -// logEventMap.put("log.level", level != null ? level.toString() : EMPTY_STRING); -// logEventMap.put("logger.name", event.getLoggerName()); -// logEventMap.put("class.name", event.getLoggerFqcn()); -// -// Throwable throwable = event.getThrown(); -// logEventMap.put("throwable", throwable != null ? throwable.toString() : EMPTY_STRING); -// -// NewRelic.getAgent().getLogSender().recordCustomEvent("LogEvent", logEventMap); -// } -// } - - public static Map getLinkingMetadataAsMap() { Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); // TODO omit attributes that don't have values From 17e5bff149fdcacb7fd0b5dce3a0ad467815d409 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 17:46:30 -0800 Subject: [PATCH 16/96] Cleanup instrumentation. Delete unused code. --- .../config/LoggerConfig_Instrumentation.java | 11 ------- .../log4j/core/impl/Log4jLogEvent.java | 29 ------------------- .../apache/logging/log4j/message/Message.java | 8 ----- .../logging/log4j/spi/AbstractLogger.java | 5 ---- .../classic/Logger_Instrumentation.java | 18 ------------ 5 files changed, 71 deletions(-) delete mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java delete mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java delete mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index c2e530d7a2..ebe1685aa9 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -17,15 +17,4 @@ protected void callAppenders(LogEvent event) { Weaver.callOriginal(); } - - -// private void processLogEvent(final LogEvent event, final LoggerConfig.LoggerConfigPredicate predicate) { -// event.setIncludeLocation(isIncludeLocation()); -// if (predicate.allow(this)) { -// callAppenders(event); -// } -// logParent(event, predicate); -// } - - } \ No newline at end of file diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java deleted file mode 100644 index 9a911249b3..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/Log4jLogEvent.java +++ /dev/null @@ -1,29 +0,0 @@ -//package org.apache.logging.log4j.core.impl; -// -//import com.newrelic.api.agent.weaver.MatchType; -//import com.newrelic.api.agent.weaver.Weave; -//import com.newrelic.api.agent.weaver.WeaveAllConstructors; -//import org.apache.logging.log4j.core.LogEvent; -//import org.apache.logging.log4j.message.Message; -// -//@Weave(originalName = "org.apache.logging.log4j.core.impl.Log4jLogEvent", type = MatchType.ExactClass) -//public class Log4jLogEvent { //implements LogEvent -// -// private Message message; -// -// @WeaveAllConstructors -// Log4jLogEvent() { -//// message = message.; -// } -// -// -// // might need to copy NewRelicContextDataProvider to instrumentation and find a way to activate it -// // https://github.com/newrelic/java-log-extensions/blob/main/log4j2/src/main/java/com/newrelic/logging/log4j2/NewRelicContextDataProvider.java -// -// -// final LogEvent event = Log4jLogEvent.newBuilder() -// -// // checkout org.apache.logging.log4j.core.layout.PatternLayout.encode -// // org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText -// -//} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java deleted file mode 100644 index 2051c98ce6..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/message/Message.java +++ /dev/null @@ -1,8 +0,0 @@ -//package org.apache.logging.log4j.message; -// -//import com.newrelic.api.agent.weaver.MatchType; -//import com.newrelic.api.agent.weaver.Weave; -// -//@Weave(originalName = "org.apache.logging.log4j.message.Message", type = MatchType.Interface) -//public interface Message { -//} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java deleted file mode 100644 index cbee943024..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/spi/AbstractLogger.java +++ /dev/null @@ -1,5 +0,0 @@ -//package org.apache.logging.log4j.spi; -// -//public abstract class AbstractLogger { -// -//} diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index 8936088678..2aeffbfb86 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -1,6 +1,5 @@ package ch.qos.logback.classic; -import ch.qos.logback.classic.Level; import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; @@ -14,23 +13,6 @@ private void buildLoggingEventAndAppend(final String localFQCN, final Marker mar final Throwable t) { NewRelic.incrementCounter("Logging/lines"); NewRelic.incrementCounter("Logging/lines/" + level); - -// msg = msg + getLinkingMetadataAsString(); - -// LoggingEvent loggingEvent = new LoggingEvent(); -// loggingEvent. - -// Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - - // TODO conditional check if logs should be decorated and sent to NR, send relevant event if so - // check if only in transaction??? - // check log level, only send up certain log levels by default? Weaver.callOriginal(); } - - // TODO look into weaving ch.qos.logback.classic.AsyncAppender, ch.qos.logback.core.encoder.Encoder, and ch.qos.logback.core.LayoutBase. - // AsyncAppender could be a spot to capture New Relic trace data - // Encoder could be a spot to convert the logs into NR JSON - // LayoutBase could be a spot to format the logs - // Look into org.slf4j.Logger as a possible common interface weave point? probably doesn't make sense } From aa5a9c58893f39c9ee829bd7a0d8c3be3d9d98e9 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 26 Jan 2022 17:59:28 -0800 Subject: [PATCH 17/96] Bit more cleanup --- .../src/main/java/com/newrelic/agent/DummyTransaction.java | 6 +++--- .../main/java/com/newrelic/agent/HarvestServiceImpl.java | 2 +- .../src/main/java/com/newrelic/agent/Transaction.java | 2 +- .../java/com/newrelic/agent/config/AgentConfigImpl.java | 4 ++-- .../java/com/newrelic/agent/transport/DataSenderImpl.java | 1 - 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java index 10ba4863c6..2847db673e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java @@ -78,7 +78,7 @@ public class DummyTransaction extends Transaction { private final Object lock = new Object(); private final Insights insights = new DummyInsights(); - private final Logs logEvents = new DummyLogEvents(); + private final Logs logs = new DummyLogs(); private final AgentConfig defaultConfig; private final TracerList tracerList = new TracerList(null, new DummySet<>()); private final TransactionTimer timer = new TransactionTimer(0); @@ -172,7 +172,7 @@ public Insights getInsightsData() { @Override public Logs getLogEventData() { - return logEvents; + return logs; } @Override @@ -656,7 +656,7 @@ public void recordCustomEvent(String eventType, Map attributes) { } } - static final class DummyLogEvents implements Logs { + static final class DummyLogs implements Logs { @Override public void recordLogEvent(Map attributes) { } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index da50db2df9..3988f0708e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -85,7 +85,7 @@ public void startHarvest(IRPMService rpmService) { public void startHarvestables(IRPMService rpmService, AgentConfig config) { Map eventHarvestConfig = config.getProperty(AgentConfigFactory.EVENT_HARVEST_CONFIG); Map spanHarvestConfig = config.getProperty(SERVER_SPAN_HARVEST_CONFIG); - // FIXME need to add log event harvest config + // FIXME need to add log event harvest config when it becomes available from the backend if (eventHarvestConfig == null) { ServiceFactory.getStatsService().doStatsWork(StatsWorks.getIncrementCounterWork( MetricNames.SUPPORTABILITY_CONNECT_MISSING_EVENT_DATA, 1)); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java index 584f1e9f75..7cfba715c4 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java @@ -192,7 +192,7 @@ public class Transaction { // Insights events added by the user during this transaction private final AtomicReference insights; - // Insights events added by the user during this transaction + // Log events added by the user during this transaction private final AtomicReference logEvents; // contains all work currently running diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java index 1911647aad..f6a44cccb0 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java @@ -735,12 +735,12 @@ private JarCollectorConfig initJarCollectorConfig() { } private InsightsConfig initInsightsConfig() { - Map props = nestedProps(LOG_SENDING); + Map props = nestedProps(CUSTOM_INSIGHT_EVENTS); return InsightsConfigImpl.createInsightsConfig(props, highSecurity); } private LogSenderConfig initLogSenderConfig() { - Map props = nestedProps(CUSTOM_INSIGHT_EVENTS); + Map props = nestedProps(LOG_SENDING); return LogSenderConfigImpl.createLogSenderConfig(props, highSecurity); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 8384fe5d4f..b4465695cb 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -334,7 +334,6 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect @Override public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { - // TODO use sendAnalyticEventsForReservoir? Or create new method to handle MELT format for log_event_data endpoint??? sendLogEventsForReservoir(CollectorMethods.LOG_EVENT_DATA, compressedEncoding, reservoirSize, eventsSeen, events); } From a731e5be2436d5aaa1f6a8dc60a7bc8726f6ef67 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 28 Jan 2022 08:36:20 -0800 Subject: [PATCH 18/96] Instrument Log4j2 ThreadContextDataProvider --- .../instrumentation/log4j2/AgentUtil.java | 27 +++++++++++++++++-- .../config/LoggerConfig_Instrumentation.java | 5 +++- ...adContextDataProvider_Instrumentation.java | 25 +++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 93dcbd000d..b039745a8d 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -7,16 +7,19 @@ import org.apache.logging.log4j.core.time.Instant; import org.apache.logging.log4j.message.Message; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; public class AgentUtil { - public static void reportNewRelicLogEvent(LogEvent event) { + public static void recordNewRelicLogEvent(LogEvent event) { final String EMPTY_STRING = ""; if (event != null) { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + Map agentLinkingMetadata = getNewRelicLinkingMetadata(); HashMap logEventMap = new HashMap<>(agentLinkingMetadata); Message message = event.getMessage(); @@ -37,4 +40,24 @@ public static void reportNewRelicLogEvent(LogEvent event) { AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } } + // TODO look into org.apache.logging.log4j.core.impl.ThreadContextDataInjector + // LogEvent actually has the linking metadata stored in the contextData when logged in LoggerConfig + // Just need to get the layout to show that data programmatically? + // Maybe this: Programmatically Modifying the Current Configuration after Initialization + // https://logging.apache.org/log4j/2.x/manual/customconfig.html + + public static Map getNewRelicLinkingMetadata() { + Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + if (linkingMetadata != null && linkingMetadata.size() > 0) { + Map map = new HashMap<>(); + Set> linkingMetadataSet = linkingMetadata.entrySet(); + for (Map.Entry entry : linkingMetadataSet) { +// map.put(NEW_RELIC_PREFIX + entry.getKey(), entry.getValue()); + map.put(entry.getKey(), entry.getValue()); + } + return map; + } else { + return Collections.emptyMap(); + } + } } diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index ebe1685aa9..5c689ef689 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -11,9 +11,12 @@ public class LoggerConfig_Instrumentation { protected void callAppenders(LogEvent event) { + // Generate log usage metrics NewRelic.incrementCounter("Logging/lines"); NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); - AgentUtil.reportNewRelicLogEvent(event); + + // Record and send LogEvent to New Relic + AgentUtil.recordNewRelicLogEvent(event); Weaver.callOriginal(); } diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java new file mode 100644 index 0000000000..2d6b3c1923 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java @@ -0,0 +1,25 @@ +package org.apache.logging.log4j.core.impl; + +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.log4j2.AgentUtil; +import org.apache.logging.log4j.ThreadContext; +import org.apache.logging.log4j.core.util.ContextDataProvider; + +import java.util.Map; + +@Weave(originalName = "org.apache.logging.log4j.core.impl.ThreadContextDataProvider", type = MatchType.ExactClass) +public class ThreadContextDataProvider_Instrumentation implements ContextDataProvider { + + @Override + public Map supplyContextData() { + // TODO check config to see if logs should be decorated or not +// if (shouldDecorateLogs) { +// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); +// } + // Inject the agent linking metadata into the Log4j2 ThreadContext + ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); + return Weaver.callOriginal(); + } +} From b0c16cdd8ff420487c1cd31307e12863885c8f21 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 31 Jan 2022 18:49:26 -0800 Subject: [PATCH 19/96] Add Log4j2 log formatting. Some refactoring. --- .../instrumentation/log4j2/AgentUtil.java | 54 ++++++++++--------- .../instrumentation/log4j2/ElementName.java | 18 +++++++ .../instrumentation/log4j2/ExceptionUtil.java | 35 ++++++++++++ ...adContextDataProvider_Instrumentation.java | 50 ++++++++--------- .../StringBuilderEncoder_Instrumentation.java | 25 +++++++++ .../spi/LoggingEvent_Instrumentation.java | 24 +-------- .../logbackclassic12/AgentUtil.java | 45 +++++++++++++--- .../logbackclassic12/ElementName.java | 18 +++++++ .../logbackclassic12/ExceptionUtil.java | 35 ++++++++++++ 9 files changed, 224 insertions(+), 80 deletions(-) create mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java create mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java create mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index b039745a8d..bc2d5b1f43 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -7,10 +7,17 @@ import org.apache.logging.log4j.core.time.Instant; import org.apache.logging.log4j.message.Message; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Set; + +import static com.nr.agent.instrumentation.log4j2.ElementName.*; +import static com.nr.agent.instrumentation.log4j2.ElementName.CLASS_NAME; +import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_CLASS; +import static com.nr.agent.instrumentation.log4j2.ElementName.LOGGER_NAME; +import static com.nr.agent.instrumentation.log4j2.ElementName.LOG_LEVEL; +import static com.nr.agent.instrumentation.log4j2.ElementName.MESSAGE; +import static com.nr.agent.instrumentation.log4j2.ElementName.THROWABLE; +import static com.nr.agent.instrumentation.log4j2.ElementName.TIMESTAMP; public class AgentUtil { @@ -18,24 +25,28 @@ public static void recordNewRelicLogEvent(LogEvent event) { final String EMPTY_STRING = ""; if (event != null) { -// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - Map agentLinkingMetadata = getNewRelicLinkingMetadata(); + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); HashMap logEventMap = new HashMap<>(agentLinkingMetadata); Message message = event.getMessage(); - logEventMap.put("message", message != null ? message.getFormattedMessage() : EMPTY_STRING); - logEventMap.put("timeStampTimeMillis", event.getTimeMillis()); + logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); + logEventMap.put(TIMESTAMP, event.getTimeMillis()); // TODO is this the correct version of the timestamp? - Instant instant = event.getInstant(); - logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); +// Instant instant = event.getInstant(); +// logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); Level level = event.getLevel(); - logEventMap.put("log.level", level != null ? level.name() : EMPTY_STRING); - logEventMap.put("logger.name", event.getLoggerName()); - logEventMap.put("class.name", event.getLoggerFqcn()); + logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); + logEventMap.put(LOGGER_NAME, event.getLoggerName()); + logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); Throwable throwable = event.getThrown(); - logEventMap.put("throwable", throwable != null ? throwable.toString() : EMPTY_STRING); + if (throwable != null) { + logEventMap.put(THROWABLE, throwable.toString()); + logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); + logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); + logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); + } AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } @@ -46,18 +57,11 @@ public static void recordNewRelicLogEvent(LogEvent event) { // Maybe this: Programmatically Modifying the Current Configuration after Initialization // https://logging.apache.org/log4j/2.x/manual/customconfig.html - public static Map getNewRelicLinkingMetadata() { - Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - if (linkingMetadata != null && linkingMetadata.size() > 0) { - Map map = new HashMap<>(); - Set> linkingMetadataSet = linkingMetadata.entrySet(); - for (Map.Entry entry : linkingMetadataSet) { -// map.put(NEW_RELIC_PREFIX + entry.getKey(), entry.getValue()); - map.put(entry.getKey(), entry.getValue()); - } - return map; - } else { - return Collections.emptyMap(); - } + public static String getLinkingMetadataAsString() { + return NewRelic.getAgent().getLinkingMetadata().toString(); + } + + public static Map getLinkingMetadataAsMap() { + return NewRelic.getAgent().getLinkingMetadata(); } } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java new file mode 100644 index 0000000000..8b95637d8e --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java @@ -0,0 +1,18 @@ +package com.nr.agent.instrumentation.log4j2; + +public class ElementName { + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String THREAD_NAME = "thread.name"; + public static final String LOG_LEVEL = "log.level"; + public static final String LOGGER_NAME = "logger.name"; + public static final String CLASS_NAME = "class.name"; + public static final String METHOD_NAME = "method.name"; + public static final String LINE_NUMBER = "line.number"; + public static final String THROWABLE = "throwable"; + public static final String ERROR_MESSAGE = "error.message"; + public static final String ERROR_CLASS = "error.class"; + public static final String ERROR_STACK = "error.stack"; + + private ElementName() {} +} diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java new file mode 100644 index 0000000000..0b073cc6e1 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019. New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nr.agent.instrumentation.log4j2; + +public class ExceptionUtil { + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit + + public static String getErrorStack(Throwable throwable) { + if (throwable == null) { + return null; + } + + StackTraceElement[] stack = throwable.getStackTrace(); + return getErrorStack(stack); + } + + public static String getErrorStack(StackTraceElement[] stack) { + return getErrorStack(stack, MAX_STACK_SIZE); + } + + public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { + if (stack == null || stack.length == 0) { + return null; + } + + StringBuilder stackBuilder = new StringBuilder(); + for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { + stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); + } + return stackBuilder.toString(); + } +} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java index 2d6b3c1923..c2e4c7e238 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java @@ -1,25 +1,25 @@ -package org.apache.logging.log4j.core.impl; - -import com.newrelic.api.agent.weaver.MatchType; -import com.newrelic.api.agent.weaver.Weave; -import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.log4j2.AgentUtil; -import org.apache.logging.log4j.ThreadContext; -import org.apache.logging.log4j.core.util.ContextDataProvider; - -import java.util.Map; - -@Weave(originalName = "org.apache.logging.log4j.core.impl.ThreadContextDataProvider", type = MatchType.ExactClass) -public class ThreadContextDataProvider_Instrumentation implements ContextDataProvider { - - @Override - public Map supplyContextData() { - // TODO check config to see if logs should be decorated or not -// if (shouldDecorateLogs) { -// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); -// } - // Inject the agent linking metadata into the Log4j2 ThreadContext - ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); - return Weaver.callOriginal(); - } -} +//package org.apache.logging.log4j.core.impl; +// +//import com.newrelic.api.agent.weaver.MatchType; +//import com.newrelic.api.agent.weaver.Weave; +//import com.newrelic.api.agent.weaver.Weaver; +//import com.nr.agent.instrumentation.log4j2.AgentUtil; +//import org.apache.logging.log4j.ThreadContext; +//import org.apache.logging.log4j.core.util.ContextDataProvider; +// +//import java.util.Map; +// +//@Weave(originalName = "org.apache.logging.log4j.core.impl.ThreadContextDataProvider", type = MatchType.ExactClass) +//public class ThreadContextDataProvider_Instrumentation implements ContextDataProvider { +// +// @Override +// public Map supplyContextData() { +// // TODO check config to see if logs should be decorated or not +//// if (shouldDecorateLogs) { +//// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); +//// } +// // Inject the agent linking metadata into the Log4j2 ThreadContext +// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); +// return Weaver.callOriginal(); +// } +//} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java new file mode 100644 index 0000000000..0536f1755b --- /dev/null +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -0,0 +1,25 @@ +package org.apache.logging.log4j.core.layout; + +import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.log4j2.AgentUtil; + +@Weave(originalName = "org.apache.logging.log4j.core.layout.StringBuilderEncoder", type = MatchType.BaseClass) +public class StringBuilderEncoder_Instrumentation { + + public void encode(final StringBuilder source, final ByteBufferDestination destination) { + // TODO check config before appending agent metadata to log file/console output. Avoid doing if possible. + appendAgentMetadata(source); + Weaver.callOriginal(); + } + + private void appendAgentMetadata(StringBuilder source) { + int breakLine = source.toString().lastIndexOf("\n"); + if (breakLine != -1) { + source.replace(breakLine, breakLine + 1, ""); + } + source.append(" NR-LINKING-METADATA: ").append(AgentUtil.getLinkingMetadataAsString()).append("\n"); + } + +} \ No newline at end of file diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 5cbb3d6693..a98fbb3483 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -3,18 +3,15 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.LoggerContext; -import com.newrelic.agent.bridge.AgentBridge; -import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; import org.slf4j.Marker; -import java.util.HashMap; import java.util.Map; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getLinkingMetadataAsMap; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) public class LoggingEvent_Instrumentation implements ILoggingEvent { @@ -31,11 +28,9 @@ public class LoggingEvent_Instrumentation implements ILoggingEvent { * The name of thread in which this logging event was generated. */ private String threadName; - private String loggerName; private LoggerContext loggerContext; private LoggerContextVO loggerContextVO; - /** * Level of logging event. *

@@ -45,14 +40,7 @@ public class LoggingEvent_Instrumentation implements ILoggingEvent { *

*/ private transient Level level; - private String message; - - // we gain significant space at serialization time by marking - // formattedMessage as transient and constructing it lazily in - // getFormattedMessage() - transient String formattedMessage; - private transient Object[] argumentArray; private ThrowableProxy throwableProxy; @@ -73,13 +61,6 @@ public LoggingEvent_Instrumentation() { } public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { - HashMap logEventMap = new HashMap<>(getLinkingMetadataAsMap()); - logEventMap.put("message", message); - logEventMap.put("timeStamp", timeStamp); - logEventMap.put("log.level", level); - logEventMap.put("logger.name", logger.getName()); - logEventMap.put("class.name", fqcn); - this.fqnOfLoggerClass = fqcn; this.loggerName = logger.getName(); this.loggerContext = logger.getLoggerContext(); @@ -97,7 +78,6 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str } if (throwable != null) { - logEventMap.put("throwable", throwable); this.throwableProxy = new ThrowableProxy(throwable); LoggerContext lc = logger.getLoggerContext(); if (lc.isPackagingDataEnabled()) { @@ -107,7 +87,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str timeStamp = System.currentTimeMillis(); - AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); + recordNewRelicLogEvent(message, timeStamp, level, logger, fqcn, throwable); } private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 2f77f4bc78..1034d003c9 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -1,22 +1,51 @@ package com.nr.agent.instrumentation.logbackclassic12; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; +import java.util.HashMap; import java.util.Map; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.CLASS_NAME; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_CLASS; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_MESSAGE; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_STACK; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.LOGGER_NAME; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.LOG_LEVEL; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.MESSAGE; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.THROWABLE; +import static com.nr.agent.instrumentation.logbackclassic12.ElementName.TIMESTAMP; + public class AgentUtil { + public static void recordNewRelicLogEvent(String message, long timeStamp, Level level, Logger logger, String fqcn, Throwable throwable) { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + HashMap logEventMap = new HashMap<>(agentLinkingMetadata); + + logEventMap.put(MESSAGE, message); + logEventMap.put(TIMESTAMP,timeStamp); + logEventMap.put(LOG_LEVEL, level); + logEventMap.put(LOGGER_NAME, logger.getName()); + logEventMap.put(CLASS_NAME, fqcn); + + if (throwable != null) { + logEventMap.put(THROWABLE, throwable.toString()); + logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); + logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); + logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); + } + + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); + } + public static Map getLinkingMetadataAsMap() { - Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - // TODO omit attributes that don't have values - // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} - return linkingMetadata; + return NewRelic.getAgent().getLinkingMetadata(); } public static String getLinkingMetadataAsString() { - Map linkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - // TODO omit attributes that don't have values - // {trace.id=, hostname=192.168.1.8, entity.type=SERVICE, entity.guid=MjIxMjg2NHxBUE18QVBQTElDQVRJT058MTY2NjIxNDQ3NQ, entity.name=SpringBoot PetClinic, span.id=} - return linkingMetadata.toString(); + return NewRelic.getAgent().getLinkingMetadata().toString(); } + } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java new file mode 100644 index 0000000000..ef6e313c47 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java @@ -0,0 +1,18 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +public class ElementName { + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String THREAD_NAME = "thread.name"; + public static final String LOG_LEVEL = "log.level"; + public static final String LOGGER_NAME = "logger.name"; + public static final String CLASS_NAME = "class.name"; + public static final String METHOD_NAME = "method.name"; + public static final String LINE_NUMBER = "line.number"; + public static final String THROWABLE = "throwable"; + public static final String ERROR_MESSAGE = "error.message"; + public static final String ERROR_CLASS = "error.class"; + public static final String ERROR_STACK = "error.stack"; + + private ElementName() {} +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java new file mode 100644 index 0000000000..f0252b189c --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019. New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nr.agent.instrumentation.logbackclassic12; + +public class ExceptionUtil { + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit + + public static String getErrorStack(Throwable throwable) { + if (throwable == null) { + return null; + } + + StackTraceElement[] stack = throwable.getStackTrace(); + return getErrorStack(stack); + } + + public static String getErrorStack(StackTraceElement[] stack) { + return getErrorStack(stack, MAX_STACK_SIZE); + } + + public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { + if (stack == null || stack.length == 0) { + return null; + } + + StringBuilder stackBuilder = new StringBuilder(); + for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { + stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); + } + return stackBuilder.toString(); + } +} From 7b4d274e8fd6de5891c0ae6b68b96ecf7643bcf5 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 31 Jan 2022 18:59:05 -0800 Subject: [PATCH 20/96] Cleanup Logback instrumentation --- .../spi/LoggingEvent_Instrumentation.java | 113 +----------------- 1 file changed, 2 insertions(+), 111 deletions(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index a98fbb3483..eb516acc47 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -7,59 +7,27 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; -import org.slf4j.Marker; - -import java.util.Map; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) -public class LoggingEvent_Instrumentation implements ILoggingEvent { - /** - * Fully qualified name of the calling Logger class. This field does not - * survive serialization. - *

- *

- * Note that the getCallerInformation() method relies on this fact. - */ - transient String fqnOfLoggerClass; +public class LoggingEvent_Instrumentation { - /** - * The name of thread in which this logging event was generated. - */ - private String threadName; + transient String fqnOfLoggerClass; private String loggerName; private LoggerContext loggerContext; private LoggerContextVO loggerContextVO; - /** - * Level of logging event. - *

- *

- * This field should not be accessed directly. You should use the - * {@link #getLevel} method instead. - *

- */ private transient Level level; private String message; private transient Object[] argumentArray; - private ThrowableProxy throwableProxy; - private StackTraceElement[] callerDataArray; - - private Marker marker; - - private Map mdcPropertyMap; - /** * The number of milliseconds elapsed from 1/1/1970 until logging event was * created. */ private long timeStamp; - public LoggingEvent_Instrumentation() { - } - public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { this.fqnOfLoggerClass = fqcn; this.loggerName = logger.getName(); @@ -94,81 +62,4 @@ private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { return Weaver.callOriginal(); } - @Override - public String getThreadName() { - return Weaver.callOriginal(); - } - - @Override - public Level getLevel() { - return Weaver.callOriginal(); - } - - @Override - public String getMessage() { - return Weaver.callOriginal(); - } - - @Override - public Object[] getArgumentArray() { - return Weaver.callOriginal(); - } - - @Override - public String getFormattedMessage() { - return Weaver.callOriginal(); - } - - @Override - public String getLoggerName() { - return Weaver.callOriginal(); - } - - @Override - public LoggerContextVO getLoggerContextVO() { - return Weaver.callOriginal(); - } - - @Override - public IThrowableProxy getThrowableProxy() { - return Weaver.callOriginal(); - } - - @Override - public StackTraceElement[] getCallerData() { - return Weaver.callOriginal(); - } - - @Override - public boolean hasCallerData() { - return Weaver.callOriginal(); - } - - @Override - public Marker getMarker() { - return Weaver.callOriginal(); - } - - @Override - public Map getMDCPropertyMap() { - return Weaver.callOriginal(); - } - - /** - * @deprecated - */ - @Override - public Map getMdc() { - return Weaver.callOriginal(); - } - - @Override - public long getTimeStamp() { - return Weaver.callOriginal(); - } - - @Override - public void prepareForDeferredProcessing() { - Weaver.callOriginal(); - } } From 694263b1921382d89d78091de39e54ce5ffcf137 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 31 Jan 2022 19:00:41 -0800 Subject: [PATCH 21/96] Cleanup Logback instrumentation --- .../logback/classic/spi/LoggingEvent_Instrumentation.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index eb516acc47..737c118b1a 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -21,11 +21,6 @@ public class LoggingEvent_Instrumentation { private String message; private transient Object[] argumentArray; private ThrowableProxy throwableProxy; - - /** - * The number of milliseconds elapsed from 1/1/1970 until logging event was - * created. - */ private long timeStamp; public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { From 2ee1344dc6a21f71176123377b4a4697a182dc4c Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 10 Feb 2022 11:07:48 -0800 Subject: [PATCH 22/96] Truncate LogEvent attribute values --- .../agent/attributes/AttributeValidator.java | 20 +++++++++++-------- .../newrelic/agent/config/ConfigConstant.java | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java index bd7e2fb3fb..31d6e65849 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributeValidator.java @@ -162,19 +162,23 @@ private boolean validateAndLogKeyLength(String key, String methodCalled) { } private String truncateValue(String key, String value, String methodCalled) { - // TODO if methodCalled is from log sender then don't truncate attribute values - // or instead set a different MAX_USER_ATTRIBUTE_SIZE if we decide to impose a limit - // Maybe create an AttributeValidator interface and a LogEventAttributeValidator with it's own truncateValue implementation instead of doing this + String truncatedVal; if (methodCalled.equals(LogSenderServiceImpl.METHOD)) { - return value; + truncatedVal = truncateString(value, ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE); + logTruncatedValue(key, value, truncatedVal, methodCalled, ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE); + } else { + truncatedVal = truncateString(value, ConfigConstant.MAX_USER_ATTRIBUTE_SIZE); + logTruncatedValue(key, value, truncatedVal, methodCalled, ConfigConstant.MAX_USER_ATTRIBUTE_SIZE); } - String truncatedVal = truncateString(value, ConfigConstant.MAX_USER_ATTRIBUTE_SIZE); + return truncatedVal; + } + + private void logTruncatedValue(String key, String value, String truncatedVal, String methodCalled, int maxAttributeSize) { if (!value.equals(truncatedVal)) { Agent.LOG.log(Level.FINER, "{0} was invoked with a value longer than {2} bytes for key \"{3}\". The value will be shortened to the first {4} characters.", - methodCalled, value, ConfigConstant.MAX_USER_ATTRIBUTE_SIZE, key, truncatedVal.length()); + methodCalled, value, maxAttributeSize, key, truncatedVal.length()); } - return truncatedVal; } /** @@ -183,7 +187,7 @@ private String truncateValue(String key, String value, String methodCalled) { * * @param s String to be truncated * @param maxBytes Maximum number of bytes in UTF-8 charset encoding - * @return + * @return truncated input string */ public static String truncateString(String s, int maxBytes) { int truncatedSize = 0; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java index 0bc7c63c77..2af3870eef 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java @@ -9,6 +9,6 @@ public class ConfigConstant { public static final int MAX_USER_ATTRIBUTES = 64; - public static final int MAX_USER_ATTRIBUTE_SIZE = 255; - public static final int MAX_LOG_EVENT_ATTRIBUTE_SIZE = 4096; // TODO what if any limit should be imposed on log event attributes? + public static final int MAX_USER_ATTRIBUTE_SIZE = 255; // Size in bytes + public static final int MAX_LOG_EVENT_ATTRIBUTE_SIZE = 32000; // Size in bytes } From d457fbe5c9ceaa521463ff77d5b48733980236c2 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 10 Feb 2022 11:34:13 -0800 Subject: [PATCH 23/96] Disable LogEvents if HSM or LASP are enabled --- .../agent/config/LogSenderConfigImpl.java | 25 +++++----- .../service/logging/LogSenderServiceImpl.java | 46 ++++++++----------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java index 3bbf32a5d9..f2e67d9793 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java @@ -10,14 +10,23 @@ import java.util.Collections; import java.util.Map; +import static com.newrelic.agent.config.AgentConfigImpl.LOG_SENDING; + public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { + public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config." + LOG_SENDING + "."; + public static final String ENABLED_PROP = "enabled"; public static final String MAX_SAMPLES_STORED_PROP = "max_samples_stored"; public static final int DEFAULT_MAX_SAMPLES_STORED = 10000; - public static final String ENABLED_PROP = "enabled"; public static final boolean DEFAULT_ENABLED = true; // TODO make off by default, add yaml config - public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config.log_sending."; public static final String ENABLED = SYSTEM_PROPERTY_ROOT + ENABLED_PROP; -// public static final String COLLECT_CUSTOM_EVENTS = "collect_custom_events"; + + /* log_sending: + * forwarding: + * enabled: true + * max_samples_stored: 10000 + * decorating: + * enabled: true + */ public final int maxSamplesStored; public final boolean isEnabled; @@ -25,20 +34,12 @@ public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { public LogSenderConfigImpl(Map pProps, boolean highSecurity) { super(pProps, SYSTEM_PROPERTY_ROOT); maxSamplesStored = getProperty(MAX_SAMPLES_STORED_PROP, DEFAULT_MAX_SAMPLES_STORED); - // TODO ignores highSecurity for the time being. Should we respect it? -// isEnabled = !highSecurity && initEnabled(); - isEnabled = initEnabled(); + isEnabled = !highSecurity && initEnabled(); } public boolean initEnabled() { boolean storedMoreThan0 = maxSamplesStored > 0; Boolean configEnabled = getProperty(ENABLED_PROP, DEFAULT_ENABLED); -// /* -// * "collect_analytics_events" is the property which comes down from the server. This gets mapped to -// * transaction_events.collect_analytics_events in AgentConfigFactory.mergeServerData() -// */ -// Boolean featureGateEnabled = getProperty(COLLECT_CUSTOM_EVENTS, DEFAULT_ENABLED); -// return storedMoreThan0 && configEnabled && featureGateEnabled; return storedMoreThan0 && configEnabled; } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 160a61f9b5..b7341e1f44 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -51,11 +51,8 @@ public class LogSenderServiceImpl extends AbstractService implements LogSenderSe private volatile boolean enabled; // Key is the app name, value is if it is enabled - should be a limited number of names private final ConcurrentMap isEnabledForApp = new ConcurrentHashMap<>(); - /* - * Number of log events in the reservoir sampling buffer per-app. All apps get the same value. - */ + // Number of log events in the reservoir sampling buffer per-app. All apps get the same value. private volatile int maxSamplesStored; - // Key is app name, value is collection of per-transaction log events for next harvest for that app. private final ConcurrentHashMap> reservoirForApp = new ConcurrentHashMap<>(); @@ -296,19 +293,15 @@ private void createAndStoreEvent(String appName, Map attributes) { */ private boolean logEventsDisabled() { if (!enabled) { - // TODO high security is disabled for now. How should we handle it? -// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { -// Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", eventType); -// } else { -// Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", eventType); -// } - - Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); - - return true; // Log Sender events are disabled + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); + } else { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); // FIXME update these logs if log_sending is not used + } + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); // FIXME update these logs if log_sending is not used + return true; // LogEvents are disabled } - - return false; // Log Sender events are enabled + return false; // LogEvents are enabled } /** @@ -385,7 +378,7 @@ public String getAppName() { } } catch (Exception e) { // discard harvest data - reservoir.clear(); + reservoir.clear(); // TODO should we simply discard all events if MaxPayloadException bubbles up to here from DataSenderImpl? Agent.LOG.log(Level.FINE, "Unable to send log events. Unsent events will be dropped.", e); } } @@ -507,12 +500,10 @@ protected String getAttributeType() { @Override protected Map getAttributeMap() { - // FIXME skip this check for now as it isn't clear what to do with Log data if LASP or high security are enabled -// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { -// return userAttributes; -// } -// return null; - return userAttributes; + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { + return userAttributes; + } + return null; } } @@ -534,11 +525,10 @@ public static final class TransactionLogs implements Logs { @Override public void recordLogEvent(Map attributes) { - // TODO ignore high security for now -// if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { -// Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); -// return; -// } + if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { + Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); + return; + } LogEvent event = createValidatedEvent(attributes); if (events.offer(event)) { From f04bdda740c9ef6380d0c11114f24508b017eb0e Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 10 Feb 2022 11:38:48 -0800 Subject: [PATCH 24/96] Clean up instrumentation a bit --- .../instrumentation/log4j2/AgentUtil.java | 15 +++-------- .../instrumentation/log4j2/ExceptionUtil.java | 2 +- .../config/LoggerConfig_Instrumentation.java | 2 +- ...adContextDataProvider_Instrumentation.java | 25 ------------------- .../classic/Logger_Instrumentation.java | 1 + .../logbackclassic12/AgentUtil.java | 5 ++-- .../logbackclassic12/ExceptionUtil.java | 2 +- 7 files changed, 11 insertions(+), 41 deletions(-) delete mode 100644 instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index bc2d5b1f43..4912452d77 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -4,15 +4,15 @@ import com.newrelic.api.agent.NewRelic; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; -import org.apache.logging.log4j.core.time.Instant; import org.apache.logging.log4j.message.Message; import java.util.HashMap; import java.util.Map; -import static com.nr.agent.instrumentation.log4j2.ElementName.*; import static com.nr.agent.instrumentation.log4j2.ElementName.CLASS_NAME; import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_CLASS; +import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_MESSAGE; +import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_STACK; import static com.nr.agent.instrumentation.log4j2.ElementName.LOGGER_NAME; import static com.nr.agent.instrumentation.log4j2.ElementName.LOG_LEVEL; import static com.nr.agent.instrumentation.log4j2.ElementName.MESSAGE; @@ -30,10 +30,7 @@ public static void recordNewRelicLogEvent(LogEvent event) { Message message = event.getMessage(); logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); - logEventMap.put(TIMESTAMP, event.getTimeMillis()); // TODO is this the correct version of the timestamp? - -// Instant instant = event.getInstant(); -// logEventMap.put("timeStampInstantEpochSecond", instant != null ? instant.getEpochSecond() : EMPTY_STRING); + logEventMap.put(TIMESTAMP, event.getTimeMillis()); Level level = event.getLevel(); logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); @@ -51,13 +48,9 @@ public static void recordNewRelicLogEvent(LogEvent event) { AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } } - // TODO look into org.apache.logging.log4j.core.impl.ThreadContextDataInjector - // LogEvent actually has the linking metadata stored in the contextData when logged in LoggerConfig - // Just need to get the layout to show that data programmatically? - // Maybe this: Programmatically Modifying the Current Configuration after Initialization - // https://logging.apache.org/log4j/2.x/manual/customconfig.html public static String getLinkingMetadataAsString() { + // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string return NewRelic.getAgent().getLinkingMetadata().toString(); } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java index 0b073cc6e1..0d6643dc3a 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java @@ -6,7 +6,7 @@ package com.nr.agent.instrumentation.log4j2; public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? public static String getErrorStack(Throwable throwable) { if (throwable == null) { diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index 5c689ef689..5c578fe690 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -11,7 +11,7 @@ public class LoggerConfig_Instrumentation { protected void callAppenders(LogEvent event) { - // Generate log usage metrics + // Generate log level metrics NewRelic.incrementCounter("Logging/lines"); NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java deleted file mode 100644 index c2e4c7e238..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/impl/ThreadContextDataProvider_Instrumentation.java +++ /dev/null @@ -1,25 +0,0 @@ -//package org.apache.logging.log4j.core.impl; -// -//import com.newrelic.api.agent.weaver.MatchType; -//import com.newrelic.api.agent.weaver.Weave; -//import com.newrelic.api.agent.weaver.Weaver; -//import com.nr.agent.instrumentation.log4j2.AgentUtil; -//import org.apache.logging.log4j.ThreadContext; -//import org.apache.logging.log4j.core.util.ContextDataProvider; -// -//import java.util.Map; -// -//@Weave(originalName = "org.apache.logging.log4j.core.impl.ThreadContextDataProvider", type = MatchType.ExactClass) -//public class ThreadContextDataProvider_Instrumentation implements ContextDataProvider { -// -// @Override -// public Map supplyContextData() { -// // TODO check config to see if logs should be decorated or not -//// if (shouldDecorateLogs) { -//// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); -//// } -// // Inject the agent linking metadata into the Log4j2 ThreadContext -// ThreadContext.putAll(AgentUtil.getNewRelicLinkingMetadata()); -// return Weaver.callOriginal(); -// } -//} diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index 2aeffbfb86..32ace19e54 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -11,6 +11,7 @@ public abstract class Logger_Instrumentation { private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { + // Generate log level metrics NewRelic.incrementCounter("Logging/lines"); NewRelic.incrementCounter("Logging/lines/" + level); Weaver.callOriginal(); diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 1034d003c9..94af52dff5 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -20,12 +20,12 @@ public class AgentUtil { - public static void recordNewRelicLogEvent(String message, long timeStamp, Level level, Logger logger, String fqcn, Throwable throwable) { + public static void recordNewRelicLogEvent(String message, long timeStampMillis, Level level, Logger logger, String fqcn, Throwable throwable) { Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); HashMap logEventMap = new HashMap<>(agentLinkingMetadata); logEventMap.put(MESSAGE, message); - logEventMap.put(TIMESTAMP,timeStamp); + logEventMap.put(TIMESTAMP, timeStampMillis); logEventMap.put(LOG_LEVEL, level); logEventMap.put(LOGGER_NAME, logger.getName()); logEventMap.put(CLASS_NAME, fqcn); @@ -45,6 +45,7 @@ public static Map getLinkingMetadataAsMap() { } public static String getLinkingMetadataAsString() { + // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string return NewRelic.getAgent().getLinkingMetadata().toString(); } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java index f0252b189c..106976d8b0 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java @@ -6,7 +6,7 @@ package com.nr.agent.instrumentation.logbackclassic12; public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? public static String getErrorStack(Throwable throwable) { if (throwable == null) { From 2b8c2a032c0c0867bf347dc4c42dc7407d7ffd40 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 10 Feb 2022 11:42:29 -0800 Subject: [PATCH 25/96] Add some TODOs --- .../src/main/java/com/newrelic/agent/RPMService.java | 3 ++- .../java/com/newrelic/agent/transport/DataSenderImpl.java | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java index 326c207320..fc09b5832d 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java @@ -411,6 +411,7 @@ public void sendErrorData(List errors) { // In case of 413 status code, cut the size of the payload in half and try again if (e.isRequestPayloadTooLarge()) { // This will halve the errors payload. If the payload only has 1 item left it will be cut to 0 + // TODO should we follow this halving behavior for sendLogEvents? sendErrorData(new ArrayList<>(errors.subList(0, errors.size() / 2))); } else { throw e; // Otherwise re-throw the error so it can be logged @@ -533,7 +534,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, final C @Override public void sendLogEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception { - Agent.LOG.log(Level.FINE, "Sending {0} Log Sender event(s)", events.size()); + Agent.LOG.log(Level.FINE, "Sending {0} log event(s)", events.size()); try { sendLogEventsSyncRestart(reservoirSize, eventsSeen, events); } catch (HttpError e) { diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index b4465695cb..6d5ade4483 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -364,13 +364,14 @@ private void sendAnalyticEventsForR // https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#log-attribute-example private void sendLogEventsForReservoir(String method, String encoding, int reservoirSize, int eventsSeen, Collection events) throws Exception { + // FIXME remove unused parameters Object runId = agentRunId; if (runId == NO_AGENT_RUN_ID || events.isEmpty()) { return; } JSONObject commonAttributes = new JSONObject(); - commonAttributes.put("plugin.type", "nr-java-agent"); + commonAttributes.put("plugin.type", "nr-java-agent"); // TODO do we set this? or does backend service? // build attributes object JSONObject attributes = new JSONObject(); @@ -570,7 +571,7 @@ private ReadResult connectAndSend(String host, String method, String encoding, S /* * We don't enforce max_payload_size_in_bytes for error_data (aka error traces). Instead we halve the - * payload and try again. See RPMService sendErrorData + * payload and try again. See RPMService sendErrorData // TODO does this apply to log data as well? */ if (data.length > maxPayloadSizeInBytes && !method.equals(CollectorMethods.ERROR_DATA)) { ServiceFactory.getStatsService().doStatsWork(StatsWorks.getIncrementCounterWork( From f53881a39f26426f82f96217557ead2ffef63521 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 15 Feb 2022 17:32:53 -0800 Subject: [PATCH 26/96] Refactor application logging config. Gate instrumentation behind config. --- .../instrumentation/log4j1/AgentUtil.java | 74 +++++++++++++ .../instrumentation/log4j1/ElementName.java | 18 ++++ .../instrumentation/log4j1/ExceptionUtil.java | 35 ++++++ .../log4j/Category_Instrumentation.java | 13 ++- .../instrumentation/log4j2/AgentUtil.java | 17 +++ .../config/LoggerConfig_Instrumentation.java | 20 ++-- .../StringBuilderEncoder_Instrumentation.java | 12 ++- .../com/nr/instrumentation/jul/AgentUtil.java | 62 +++++++++++ .../nr/instrumentation/jul/ElementName.java | 18 ++++ .../nr/instrumentation/jul/ExceptionUtil.java | 35 ++++++ .../jul/Logger_Instrumentation.java | 30 ------ .../util/logging/Logger_Instrumentation.java | 39 +++++++ .../classic/Logger_Instrumentation.java | 12 ++- .../spi/LoggingEvent_Instrumentation.java | 58 ++++++---- .../logbackclassic12/AgentUtil.java | 15 +++ .../newrelic/agent/HarvestServiceImpl.java | 5 +- .../newrelic/agent/config/AgentConfig.java | 6 +- .../agent/config/AgentConfigImpl.java | 16 +-- .../config/ApplicationLoggingConfig.java | 50 +++++++++ .../config/ApplicationLoggingConfigImpl.java | 100 ++++++++++++++++++ .../ApplicationLoggingForwardingConfig.java | 37 +++++++ ...plicationLoggingLocalDecoratingConfig.java | 28 +++++ .../ApplicationLoggingMetricsConfig.java | 27 +++++ .../agent/config/LogSenderConfig.java | 19 ---- .../agent/config/LogSenderConfigImpl.java | 61 ----------- .../agent/service/ServiceFactory.java | 5 + .../logging/LogSenderHarvestableImpl.java | 2 +- .../service/logging/LogSenderServiceImpl.java | 11 +- .../src/main/resources/newrelic.yml | 13 +++ 29 files changed, 674 insertions(+), 164 deletions(-) create mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java create mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java create mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java create mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java create mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java create mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java delete mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java create mode 100644 instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java delete mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java delete mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java new file mode 100644 index 0000000000..10f786b8f0 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -0,0 +1,74 @@ +package com.nr.agent.instrumentation.log4j1; + +import com.newrelic.agent.bridge.AgentBridge; +import com.newrelic.api.agent.NewRelic; + +import java.util.HashMap; +import java.util.Map; + +import static com.nr.agent.instrumentation.log4j1.ElementName.CLASS_NAME; +import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_CLASS; +import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_MESSAGE; +import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_STACK; +import static com.nr.agent.instrumentation.log4j1.ElementName.LOGGER_NAME; +import static com.nr.agent.instrumentation.log4j1.ElementName.LOG_LEVEL; +import static com.nr.agent.instrumentation.log4j1.ElementName.MESSAGE; +import static com.nr.agent.instrumentation.log4j1.ElementName.THROWABLE; +import static com.nr.agent.instrumentation.log4j1.ElementName.TIMESTAMP; + +public class AgentUtil { + + // TODO figure out how to recordNewRelicLogEvent for Log4j1 +// public static void recordNewRelicLogEvent(LogEvent event) { +// final String EMPTY_STRING = ""; +// +// if (event != null) { +// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); +// +// Message message = event.getMessage(); +// logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); +// logEventMap.put(TIMESTAMP, event.getTimeMillis()); +// +// Level level = event.getLevel(); +// logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); +// logEventMap.put(LOGGER_NAME, event.getLoggerName()); +// logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); +// +// Throwable throwable = event.getThrown(); +// if (throwable != null) { +// logEventMap.put(THROWABLE, throwable.toString()); +// logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); +// logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); +// logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); +// } +// +// AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); +// } +// } + + public static String getLinkingMetadataAsString() { + // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string + return NewRelic.getAgent().getLinkingMetadata().toString(); + } + + public static Map getLinkingMetadataAsMap() { + return NewRelic.getAgent().getLinkingMetadata(); + } + + public static boolean isApplicationLoggingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + } + + public static boolean isApplicationLoggingMetricsEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + } + + public static boolean isApplicationLoggingForwardingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + } + + public static boolean isApplicationLoggingLocalDecoratingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + } +} diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java new file mode 100644 index 0000000000..3b02ccd649 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java @@ -0,0 +1,18 @@ +package com.nr.agent.instrumentation.log4j1; + +public class ElementName { + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String THREAD_NAME = "thread.name"; + public static final String LOG_LEVEL = "log.level"; + public static final String LOGGER_NAME = "logger.name"; + public static final String CLASS_NAME = "class.name"; + public static final String METHOD_NAME = "method.name"; + public static final String LINE_NUMBER = "line.number"; + public static final String THROWABLE = "throwable"; + public static final String ERROR_MESSAGE = "error.message"; + public static final String ERROR_CLASS = "error.class"; + public static final String ERROR_STACK = "error.stack"; + + private ElementName() {} +} diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java new file mode 100644 index 0000000000..bf90224007 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019. New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nr.agent.instrumentation.log4j1; + +public class ExceptionUtil { + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? + + public static String getErrorStack(Throwable throwable) { + if (throwable == null) { + return null; + } + + StackTraceElement[] stack = throwable.getStackTrace(); + return getErrorStack(stack); + } + + public static String getErrorStack(StackTraceElement[] stack) { + return getErrorStack(stack, MAX_STACK_SIZE); + } + + public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { + if (stack == null || stack.length == 0) { + return null; + } + + StringBuilder stackBuilder = new StringBuilder(); + for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { + stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); + } + return stackBuilder.toString(); + } +} diff --git a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java index d7c8c05f9f..ceaf8419d6 100644 --- a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java @@ -3,14 +3,23 @@ import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.log4j1.AgentUtil; import org.apache.log4j.Priority; +import static com.nr.agent.instrumentation.log4j1.AgentUtil.*; + @Weave(originalName = "org.apache.log4j.Category") public class Category_Instrumentation { protected void forcedLog(String fqcn, Priority level, Object message, Throwable t) { - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + level.toString()); + // Do nothing if application_logging.enabled: false + if (isApplicationLoggingEnabled()) { + if (isApplicationLoggingMetricsEnabled()) { + // Generate log level metrics + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + level.toString()); + } + } Weaver.callOriginal(); } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 4912452d77..1dff2b98a8 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -1,6 +1,7 @@ package com.nr.agent.instrumentation.log4j2; import com.newrelic.agent.bridge.AgentBridge; +import com.newrelic.api.agent.Config; import com.newrelic.api.agent.NewRelic; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; @@ -57,4 +58,20 @@ public static String getLinkingMetadataAsString() { public static Map getLinkingMetadataAsMap() { return NewRelic.getAgent().getLinkingMetadata(); } + + public static boolean isApplicationLoggingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + } + + public static boolean isApplicationLoggingMetricsEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + } + + public static boolean isApplicationLoggingForwardingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + } + + public static boolean isApplicationLoggingLocalDecoratingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + } } diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index 5c578fe690..a295592268 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -7,17 +7,25 @@ import com.nr.agent.instrumentation.log4j2.AgentUtil; import org.apache.logging.log4j.core.LogEvent; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.*; + @Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig", type = MatchType.ExactClass) public class LoggerConfig_Instrumentation { protected void callAppenders(LogEvent event) { - // Generate log level metrics - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); - - // Record and send LogEvent to New Relic - AgentUtil.recordNewRelicLogEvent(event); + // Do nothing if application_logging.enabled: false + if (isApplicationLoggingEnabled()) { + if (isApplicationLoggingMetricsEnabled()) { + // Generate log level metrics + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + event.getLevel().toString()); + } + if (isApplicationLoggingForwardingEnabled()) { + // Record and send LogEvent to New Relic + recordNewRelicLogEvent(event); + } + } Weaver.callOriginal(); } } \ No newline at end of file diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java index 0536f1755b..cc5105988f 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -5,12 +5,20 @@ import com.newrelic.api.agent.weaver.Weaver; import com.nr.agent.instrumentation.log4j2.AgentUtil; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; + @Weave(originalName = "org.apache.logging.log4j.core.layout.StringBuilderEncoder", type = MatchType.BaseClass) public class StringBuilderEncoder_Instrumentation { public void encode(final StringBuilder source, final ByteBufferDestination destination) { - // TODO check config before appending agent metadata to log file/console output. Avoid doing if possible. - appendAgentMetadata(source); + // Do nothing if application_logging.enabled: false + if (isApplicationLoggingEnabled()) { + if (isApplicationLoggingLocalDecoratingEnabled()) { + // Append New Relic linking metadata from agent to log message + appendAgentMetadata(source); + } + } Weaver.callOriginal(); } diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java new file mode 100644 index 0000000000..f9a6fdc05b --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -0,0 +1,62 @@ +package com.nr.instrumentation.jul; + +import com.newrelic.api.agent.NewRelic; + +import java.util.Map; + +public class AgentUtil { + + // TODO figure out how to recordNewRelicLogEvent for JUL +// public static void recordNewRelicLogEvent(LogEvent event) { +// final String EMPTY_STRING = ""; +// +// if (event != null) { +// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); +// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); +// +// Message message = event.getMessage(); +// logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); +// logEventMap.put(TIMESTAMP, event.getTimeMillis()); +// +// Level level = event.getLevel(); +// logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); +// logEventMap.put(LOGGER_NAME, event.getLoggerName()); +// logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); +// +// Throwable throwable = event.getThrown(); +// if (throwable != null) { +// logEventMap.put(THROWABLE, throwable.toString()); +// logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); +// logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); +// logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); +// } +// +// AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); +// } +// } + + public static String getLinkingMetadataAsString() { + // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string + return NewRelic.getAgent().getLinkingMetadata().toString(); + } + + public static Map getLinkingMetadataAsMap() { + return NewRelic.getAgent().getLinkingMetadata(); + } + + public static boolean isApplicationLoggingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + } + + public static boolean isApplicationLoggingMetricsEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + } + + public static boolean isApplicationLoggingForwardingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + } + + public static boolean isApplicationLoggingLocalDecoratingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + } +} diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java new file mode 100644 index 0000000000..fd16eb473a --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java @@ -0,0 +1,18 @@ +package com.nr.instrumentation.jul; + +public class ElementName { + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String THREAD_NAME = "thread.name"; + public static final String LOG_LEVEL = "log.level"; + public static final String LOGGER_NAME = "logger.name"; + public static final String CLASS_NAME = "class.name"; + public static final String METHOD_NAME = "method.name"; + public static final String LINE_NUMBER = "line.number"; + public static final String THROWABLE = "throwable"; + public static final String ERROR_MESSAGE = "error.message"; + public static final String ERROR_CLASS = "error.class"; + public static final String ERROR_STACK = "error.stack"; + + private ElementName() {} +} diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java new file mode 100644 index 0000000000..43e3705b9a --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019. New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nr.instrumentation.jul; + +public class ExceptionUtil { + public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? + + public static String getErrorStack(Throwable throwable) { + if (throwable == null) { + return null; + } + + StackTraceElement[] stack = throwable.getStackTrace(); + return getErrorStack(stack); + } + + public static String getErrorStack(StackTraceElement[] stack) { + return getErrorStack(stack, MAX_STACK_SIZE); + } + + public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { + if (stack == null || stack.length == 0) { + return null; + } + + StringBuilder stackBuilder = new StringBuilder(); + for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { + stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); + } + return stackBuilder.toString(); + } +} diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java deleted file mode 100644 index 3cadb1c0bd..0000000000 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/Logger_Instrumentation.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.nr.instrumentation.jul; - -import com.newrelic.api.agent.NewRelic; -import com.newrelic.api.agent.weaver.Weave; -import com.newrelic.api.agent.weaver.Weaver; - -import java.util.logging.Filter; -import java.util.logging.Level; -import java.util.logging.LogRecord; - -@Weave(originalName = "java.util.logging.Logger") -public class Logger_Instrumentation { - - public Filter getFilter() { - return Weaver.callOriginal(); - } - - public boolean isLoggable(Level level) { - return Boolean.TRUE.equals(Weaver.callOriginal()); - } - - public void log(LogRecord record) { - if (isLoggable(record.getLevel()) && getFilter() == null || getFilter().isLoggable(record)) { - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + record.getLevel().toString()); - } - Weaver.callOriginal(); - } - -} diff --git a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java new file mode 100644 index 0000000000..47c0573067 --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java @@ -0,0 +1,39 @@ +package java.util.logging; + +import com.newrelic.api.agent.NewRelic; +import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.Weaver; +import com.nr.instrumentation.jul.AgentUtil; + +import java.util.logging.Filter; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import static com.nr.instrumentation.jul.AgentUtil.*; + +@Weave(originalName = "java.util.logging.Logger") +public class Logger_Instrumentation { + + public Filter getFilter() { + return Weaver.callOriginal(); + } + + public boolean isLoggable(Level level) { + return Boolean.TRUE.equals(Weaver.callOriginal()); + } + + public void log(LogRecord record) { + // Do nothing if application_logging.enabled: false + if (isApplicationLoggingEnabled()) { + if (isApplicationLoggingMetricsEnabled()) { + if (isLoggable(record.getLevel()) && getFilter() == null || getFilter().isLoggable(record)) { + // Generate log level metrics + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + record.getLevel().toString()); + } + } + } + Weaver.callOriginal(); + } + +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index 32ace19e54..2356036136 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -4,6 +4,7 @@ import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; +import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; import org.slf4j.Marker; @Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.ExactClass) @@ -11,9 +12,14 @@ public abstract class Logger_Instrumentation { private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { - // Generate log level metrics - NewRelic.incrementCounter("Logging/lines"); - NewRelic.incrementCounter("Logging/lines/" + level); + // Do nothing if application_logging.enabled: false + if (AgentUtil.isApplicationLoggingEnabled()) { + if (AgentUtil.isApplicationLoggingMetricsEnabled()) { + // Generate log level metrics + NewRelic.incrementCounter("Logging/lines"); + NewRelic.incrementCounter("Logging/lines/" + level); + } + } Weaver.callOriginal(); } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 737c118b1a..58aa882bfd 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -8,6 +8,7 @@ import com.newrelic.api.agent.weaver.Weaver; import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.*; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) @@ -24,33 +25,44 @@ public class LoggingEvent_Instrumentation { private long timeStamp; public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { - this.fqnOfLoggerClass = fqcn; - this.loggerName = logger.getName(); - this.loggerContext = logger.getLoggerContext(); - this.loggerContextVO = loggerContext.getLoggerContextRemoteView(); - this.level = level; - - // Append New Relic linking metadata from agent to log message - // TODO conditional checks based on config - // Should log be decorated? Should the decoration persist in the log file/console? Only log at certain log level? - this.message = message + " NR-LINKING-METADATA: " + AgentUtil.getLinkingMetadataAsString(); - this.argumentArray = argArray; - - if (throwable == null) { - throwable = extractThrowableAnRearrangeArguments(argArray); - } + // Do nothing if application_logging.enabled: false + if (isApplicationLoggingEnabled()) { + this.fqnOfLoggerClass = fqcn; + this.loggerName = logger.getName(); + this.loggerContext = logger.getLoggerContext(); + this.loggerContextVO = loggerContext.getLoggerContextRemoteView(); + this.level = level; - if (throwable != null) { - this.throwableProxy = new ThrowableProxy(throwable); - LoggerContext lc = logger.getLoggerContext(); - if (lc.isPackagingDataEnabled()) { - this.throwableProxy.calculatePackagingData(); + if (isApplicationLoggingLocalDecoratingEnabled()) { + // Append New Relic linking metadata from agent to log message + this.message = message + " NR-LINKING-METADATA: " + getLinkingMetadataAsString(); + } else { + this.message = message; } - } - timeStamp = System.currentTimeMillis(); + this.argumentArray = argArray; - recordNewRelicLogEvent(message, timeStamp, level, logger, fqcn, throwable); + if (throwable == null) { + throwable = extractThrowableAnRearrangeArguments(argArray); + } + + if (throwable != null) { + this.throwableProxy = new ThrowableProxy(throwable); + LoggerContext lc = logger.getLoggerContext(); + if (lc.isPackagingDataEnabled()) { + this.throwableProxy.calculatePackagingData(); + } + } + + timeStamp = System.currentTimeMillis(); + + if (isApplicationLoggingForwardingEnabled()) { + // Record and send LogEvent to New Relic + recordNewRelicLogEvent(message, timeStamp, level, logger, fqcn, throwable); + } + } else { + Weaver.callOriginal(); + } } private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 94af52dff5..1b99685384 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -49,4 +49,19 @@ public static String getLinkingMetadataAsString() { return NewRelic.getAgent().getLinkingMetadata().toString(); } + public static boolean isApplicationLoggingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + } + + public static boolean isApplicationLoggingMetricsEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + } + + public static boolean isApplicationLoggingForwardingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + } + + public static boolean isApplicationLoggingLocalDecoratingEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index 3988f0708e..af06f0e8f6 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -105,9 +105,10 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); // TODO set harvest_limits for log_event_data endpoint Long harvestLimit; - // TODO THIS IS A HACK! Real limit for log_event_data endpoint should come from server side + // TODO THIS IS A HACK! Real limit for log_event_data endpoint should come from server side!!! if (isLogEventEndpoint) { - harvestLimit = 1000L; + int maxSamplesStoredLogEvents = ServiceFactory.getLogSenderService().getMaxSamplesStored(); + harvestLimit = (long) maxSamplesStoredLogEvents; } else { harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java index 5c30b09c7a..9d3b109086 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfig.java @@ -173,9 +173,11 @@ public interface AgentConfig extends com.newrelic.api.agent.Config, DataSenderCo InsightsConfig getInsightsConfig(); /** - * Get the Log Sender configuration. + * Get the application logging configuration. + * + * @return ApplicationLoggingConfig used by LogSenderService */ - LogSenderConfig getLogSenderConfig(); + ApplicationLoggingConfig getApplicationLoggingConfig(); /** * Get the attributes configuration. diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java index f6a44cccb0..7c36b2341a 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java @@ -102,6 +102,7 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig { public static final String KEY_TRANSACTIONS = "web_transactions_apdex"; // nested configs (alphabetized) + public static final String APPLICATION_LOGGING = "application_logging"; public static final String ATTRIBUTES = "attributes"; public static final String BROWSER_MONITORING = "browser_monitoring"; public static final String CLASS_TRANSFORMER = "class_transformer"; @@ -115,7 +116,6 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig { public static final String JAR_COLLECTOR = "jar_collector"; public static final String JMX = "jmx"; public static final String JFR = "jfr"; - public static final String LOG_SENDING = "log_sending"; public static final String OPEN_TRACING = "open_tracing"; public static final String REINSTRUMENT = "reinstrument"; public static final String SLOW_SQL = "slow_sql"; @@ -254,7 +254,7 @@ public class AgentConfigImpl extends BaseConfig implements AgentConfig { private final ExternalTracerConfig externalTracerConfig; private final InfiniteTracingConfig infiniteTracingConfig; private final InsightsConfig insightsConfig; - private final LogSenderConfig logSenderConfig; + private final ApplicationLoggingConfig applicationLoggingConfig; private final Config instrumentationConfig; private final JarCollectorConfig jarCollectorConfig; private final JfrConfig jfrConfig; @@ -354,7 +354,7 @@ private AgentConfigImpl(Map props) { jmxConfig = initJmxConfig(); jarCollectorConfig = initJarCollectorConfig(); insightsConfig = initInsightsConfig(); - logSenderConfig = initLogSenderConfig(); + applicationLoggingConfig = initApplicationLoggingConfig(); infiniteTracingConfig = initInfiniteTracingConfig(autoAppNamingEnabled); attributesConfig = initAttributesConfig(); reinstrumentConfig = initReinstrumentConfig(); @@ -739,9 +739,9 @@ private InsightsConfig initInsightsConfig() { return InsightsConfigImpl.createInsightsConfig(props, highSecurity); } - private LogSenderConfig initLogSenderConfig() { - Map props = nestedProps(LOG_SENDING); - return LogSenderConfigImpl.createLogSenderConfig(props, highSecurity); + private ApplicationLoggingConfig initApplicationLoggingConfig() { + Map props = nestedProps(APPLICATION_LOGGING); + return ApplicationLoggingConfigImpl.createApplicationLoggingConfig(props, highSecurity); } private AttributesConfig initAttributesConfig() { @@ -1192,8 +1192,8 @@ public InsightsConfig getInsightsConfig() { } @Override - public LogSenderConfig getLogSenderConfig() { - return logSenderConfig; + public ApplicationLoggingConfig getApplicationLoggingConfig() { + return applicationLoggingConfig; } @Override diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java new file mode 100644 index 0000000000..b72bf1a7e9 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java @@ -0,0 +1,50 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +/** + * Configuration for application logging features. These settings do not pertain to agent logs. + */ +public interface ApplicationLoggingConfig { + + /** + * Determines whether the application_logging features are completely disabled or can be controlled individually. + * + * @return true if the application_logging features can be controlled individually, false if the entire stanza is disabled + */ + boolean isEnabled(); + + /** + * Allow metrics to be generated to provide data such as the number of lines logged at each log level. + * + * @return true is log metrics are enabled, otherwise false + */ + boolean isMetricsEnabled(); + + /** + * Allow the agent to forward application logs to New Relic. + * + * @return true is log forwarding is enabled, otherwise false + */ + boolean isForwardingEnabled(); + + /** + * Allow the agent to decorate application log files and console output with New Relic specific linking metadata. + * + * @return true is local log decorating is enabled, otherwise false + */ + boolean isLocalDecoratingEnabled(); + + /** + * Get the max number of LogEvents that can be stored during a harvest period before sampling takes place. + * + * @return max number of LogEvents stored per harvest + */ + int getMaxSamplesStored(); + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java new file mode 100644 index 0000000000..088b20bc67 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java @@ -0,0 +1,100 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import java.util.Collections; +import java.util.Map; + +import static com.newrelic.agent.config.AgentConfigImpl.APPLICATION_LOGGING; + +/* Default config should look like: + * + * application_logging: + * enabled: false + * forwarding: + * enabled: false + * max_samples_stored: 2000 + * local_decorating: + * enabled: false + * metrics: + * enabled: false + */ +public class ApplicationLoggingConfigImpl extends BaseConfig implements ApplicationLoggingConfig { + public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config." + APPLICATION_LOGGING + "."; + public static final String METRICS = "metrics"; + public static final String FORWARDING = "forwarding"; + public static final String LOCAL_DECORATING = "local_decorating"; + + public static final boolean DEFAULT_ENABLED = false; // TODO what is default? + public static final String ENABLED = "enabled"; + + private final ApplicationLoggingMetricsConfig applicationLoggingMetricsConfig; + private final ApplicationLoggingLocalDecoratingConfig applicationLoggingLocalDecoratingConfig; + private final ApplicationLoggingForwardingConfig applicationLoggingForwardingConfig; + + private final boolean applicationLoggingEnabled; + + public ApplicationLoggingConfigImpl(Map pProps, boolean highSecurity) { + super(pProps, SYSTEM_PROPERTY_ROOT); + applicationLoggingEnabled = getProperty(ENABLED, DEFAULT_ENABLED); + applicationLoggingMetricsConfig = createApplicationLoggingMetricsConfig(); + applicationLoggingLocalDecoratingConfig = createApplicationLoggingLocalDecoratingConfig(); + applicationLoggingForwardingConfig = createApplicationLoggingForwardingConfig(highSecurity); + } + + private ApplicationLoggingMetricsConfig createApplicationLoggingMetricsConfig() { + Map metricsProps = getProperty(METRICS, Collections.emptyMap()); + return new ApplicationLoggingMetricsConfig(metricsProps, SYSTEM_PROPERTY_ROOT); + } + + private ApplicationLoggingLocalDecoratingConfig createApplicationLoggingLocalDecoratingConfig() { + Map localDecoratingProps = getProperty(LOCAL_DECORATING, Collections.emptyMap()); + return new ApplicationLoggingLocalDecoratingConfig(localDecoratingProps, SYSTEM_PROPERTY_ROOT); + } + + private ApplicationLoggingForwardingConfig createApplicationLoggingForwardingConfig(boolean highSecurity) { + Map forwardingProps = getProperty(FORWARDING, Collections.emptyMap()); + return new ApplicationLoggingForwardingConfig(forwardingProps, SYSTEM_PROPERTY_ROOT, highSecurity); + } + +// public boolean initForwardingEnabled(boolean highSecurity) { +// boolean storedMoreThan0 = maxSamplesStored > 0; +// Boolean configEnabled = getProperty(FORWARDING_ENABLED_PROP, DEFAULT_FORWARDING_ENABLED); +// return isEnabled && !highSecurity && storedMoreThan0 && configEnabled; +// } + + static ApplicationLoggingConfigImpl createApplicationLoggingConfig(Map settings, boolean highSecurity) { + if (settings == null) { + settings = Collections.emptyMap(); + } + return new ApplicationLoggingConfigImpl(settings, highSecurity); + } + + public boolean isEnabled() { + return applicationLoggingEnabled; + } + + @Override + public boolean isMetricsEnabled() { + return applicationLoggingEnabled && applicationLoggingMetricsConfig.getEnabled(); + } + + @Override + public boolean isLocalDecoratingEnabled() { + return applicationLoggingEnabled && applicationLoggingLocalDecoratingConfig.getEnabled(); + } + + @Override + public boolean isForwardingEnabled() { + return applicationLoggingEnabled && applicationLoggingForwardingConfig.getEnabled(); + } + + public int getMaxSamplesStored() { + return applicationLoggingForwardingConfig.getMaxSamplesStored(); + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java new file mode 100644 index 0000000000..75b1b0b9d1 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java @@ -0,0 +1,37 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import java.util.Map; + +public class ApplicationLoggingForwardingConfig extends BaseConfig { + public static final String ROOT = "forwarding"; + public static final String ENABLED = "enabled"; + public static final String MAX_SAMPLES_STORED = "max_samples_stored"; + + public static final boolean DEFAULT_ENABLED = false; + public static final int DEFAULT_MAX_SAMPLES_STORED = 2000; + + private final boolean enabled; + private final int maxSamplesStored; + + public ApplicationLoggingForwardingConfig(Map props, String parentRoot, boolean highSecurity) { + super(props, parentRoot + ROOT + "."); + maxSamplesStored = getProperty(MAX_SAMPLES_STORED, DEFAULT_MAX_SAMPLES_STORED); + boolean storedMoreThan0 = maxSamplesStored > 0; + enabled = storedMoreThan0 && !highSecurity && getProperty(ENABLED, DEFAULT_ENABLED); + } + + public boolean getEnabled() { + return enabled; + } + + public int getMaxSamplesStored() { + return maxSamplesStored; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java new file mode 100644 index 0000000000..b27e98df46 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java @@ -0,0 +1,28 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import java.util.Map; + +public class ApplicationLoggingLocalDecoratingConfig extends BaseConfig { + public static final String ROOT = "local_decorating"; + public static final String ENABLED = "enabled"; + + public static final boolean DEFAULT_ENABLED = false; + + private final boolean enabled; + + public ApplicationLoggingLocalDecoratingConfig(Map props, String parentRoot) { + super(props, parentRoot + ROOT + "."); + enabled = getProperty(ENABLED, DEFAULT_ENABLED); + } + + public boolean getEnabled() { + return enabled; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java new file mode 100644 index 0000000000..1433ba1b57 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import java.util.Map; + +public class ApplicationLoggingMetricsConfig extends BaseConfig { + public static final String ROOT = "metrics"; + public static final String ENABLED = "enabled"; + public static final boolean DEFAULT_ENABLED = false; + + private final boolean enabled; + + public ApplicationLoggingMetricsConfig(Map props, String parentRoot) { + super(props, parentRoot + ROOT + "."); + enabled = getProperty(ENABLED, DEFAULT_ENABLED); + } + + public boolean getEnabled() { + return enabled; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java deleted file mode 100644 index 6c197b2fc2..0000000000 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * - * * Copyright 2020 New Relic Corporation. All rights reserved. - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package com.newrelic.agent.config; - -/** - * Configuration got LogSenderService - */ -public interface LogSenderConfig { - - boolean isEnabled(); - - int getMaxSamplesStored(); - -} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java deleted file mode 100644 index f2e67d9793..0000000000 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/LogSenderConfigImpl.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * * Copyright 2020 New Relic Corporation. All rights reserved. - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package com.newrelic.agent.config; - -import java.util.Collections; -import java.util.Map; - -import static com.newrelic.agent.config.AgentConfigImpl.LOG_SENDING; - -public class LogSenderConfigImpl extends BaseConfig implements LogSenderConfig { - public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config." + LOG_SENDING + "."; - public static final String ENABLED_PROP = "enabled"; - public static final String MAX_SAMPLES_STORED_PROP = "max_samples_stored"; - public static final int DEFAULT_MAX_SAMPLES_STORED = 10000; - public static final boolean DEFAULT_ENABLED = true; // TODO make off by default, add yaml config - public static final String ENABLED = SYSTEM_PROPERTY_ROOT + ENABLED_PROP; - - /* log_sending: - * forwarding: - * enabled: true - * max_samples_stored: 10000 - * decorating: - * enabled: true - */ - - public final int maxSamplesStored; - public final boolean isEnabled; - - public LogSenderConfigImpl(Map pProps, boolean highSecurity) { - super(pProps, SYSTEM_PROPERTY_ROOT); - maxSamplesStored = getProperty(MAX_SAMPLES_STORED_PROP, DEFAULT_MAX_SAMPLES_STORED); - isEnabled = !highSecurity && initEnabled(); - } - - public boolean initEnabled() { - boolean storedMoreThan0 = maxSamplesStored > 0; - Boolean configEnabled = getProperty(ENABLED_PROP, DEFAULT_ENABLED); - return storedMoreThan0 && configEnabled; - } - - static LogSenderConfigImpl createLogSenderConfig(Map settings, boolean highSecurity) { - if (settings == null) { - settings = Collections.emptyMap(); - } - return new LogSenderConfigImpl(settings, highSecurity); - } - - public boolean isEnabled() { - return isEnabled; - } - - public int getMaxSamplesStored() { - return maxSamplesStored; - } - -} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceFactory.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceFactory.java index 8ce469ee5b..0ba3b1ca99 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceFactory.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/ServiceFactory.java @@ -35,6 +35,7 @@ import com.newrelic.agent.service.analytics.SpanEventsService; import com.newrelic.agent.service.analytics.TransactionEventsService; import com.newrelic.agent.service.async.AsyncTransactionService; +import com.newrelic.agent.service.logging.LogSenderService; import com.newrelic.agent.service.module.JarCollectorService; import com.newrelic.agent.sql.SqlTraceService; import com.newrelic.agent.stats.StatsService; @@ -78,6 +79,10 @@ public static ConfigService getConfigService() { return SERVICE_MANAGER.getConfigService(); } + public static LogSenderService getLogSenderService() { + return SERVICE_MANAGER.getLogSenderService(); + } + public static RPMConnectionService getRPMConnectionService() { return SERVICE_MANAGER.getRPMConnectionService(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java index e102f10932..68d2a89d53 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java @@ -34,6 +34,6 @@ public String getEndpointMethodName() { */ @Override public int getMaxSamplesStored() { - return ServiceFactory.getConfigService().getDefaultAgentConfig().getLogSenderConfig().getMaxSamplesStored(); + return ServiceFactory.getConfigService().getDefaultAgentConfig().getApplicationLoggingConfig().getMaxSamplesStored(); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index b7341e1f44..fdae92d2dc 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -94,7 +94,8 @@ public void dispatcherTransactionCancelled(Transaction transaction) { public void configChanged(String appName, AgentConfig agentConfig) { // if the config has changed for the app, just remove it and regenerate enabled next transaction isEnabledForApp.remove(appName); - enabled = agentConfig.getLogSenderConfig().isEnabled(); + enabled = agentConfig.getApplicationLoggingConfig().isForwardingEnabled(); + maxSamplesStored = agentConfig.getApplicationLoggingConfig().getMaxSamplesStored(); } }; @@ -103,8 +104,8 @@ public void configChanged(String appName, AgentConfig agentConfig) { public LogSenderServiceImpl() { super(LogSenderServiceImpl.class.getSimpleName()); AgentConfig config = ServiceFactory.getConfigService().getDefaultAgentConfig(); - maxSamplesStored = config.getLogSenderConfig().getMaxSamplesStored(); - enabled = config.getLogSenderConfig().isEnabled(); + maxSamplesStored = config.getApplicationLoggingConfig().getMaxSamplesStored(); + enabled = config.getApplicationLoggingConfig().isForwardingEnabled(); isEnabledForApp.put(config.getApplicationName(), enabled); } @@ -412,7 +413,7 @@ private void recordSupportabilityMetrics(StatsEngine statsEngine, long durationI private boolean getIsEnabledForApp(AgentConfig config, String currentAppName) { Boolean appEnabled = currentAppName == null ? null : isEnabledForApp.get(currentAppName); if (appEnabled == null) { - appEnabled = config.getLogSenderConfig().isEnabled(); + appEnabled = config.getApplicationLoggingConfig().isForwardingEnabled(); isEnabledForApp.put(currentAppName, appEnabled); } return appEnabled; @@ -519,7 +520,7 @@ public static final class TransactionLogs implements Logs { final LinkedBlockingQueue events; TransactionLogs(AgentConfig config) { - int maxSamplesStored = config.getLogSenderConfig().getMaxSamplesStored(); + int maxSamplesStored = config.getApplicationLoggingConfig().getMaxSamplesStored(); events = new LinkedBlockingQueue<>(maxSamplesStored); } diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 8c1be7c8c8..63f41a5cb0 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -83,6 +83,19 @@ common: &default_settings # Default is the logs directory in the newrelic.jar parent directory. #log_file_path: + # Provides the ability to have the agent forward application logs to New Relic + # in a format that is enhanced with agent metadata. This allows the logs to be + # linked with specific transactions and errors. + application_logging: + enabled: false + forwarding: + enabled: false + max_samples_stored: 2000 + local_decorating: + enabled: false + metrics: + enabled: false + # Proxy settings for connecting to the New Relic server: # If a proxy is used, the host setting is required. Other settings # are optional. Default port is 8080. The username and password From 0fcf9c264b69b49b96b40cea0c6318c457115a4e Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 4 Feb 2022 18:37:43 -0800 Subject: [PATCH 27/96] WIP. Add plumbing for recording data usage metrics. --- .../java/com/newrelic/agent/MetricNames.java | 6 + .../newrelic/agent/stats/DataUsageStats.java | 44 ++++++ .../agent/stats/DataUsageStatsImpl.java | 136 ++++++++++++++++++ .../agent/stats/RecordDataUsageMetric.java | 33 +++++ .../agent/stats/SimpleStatsEngine.java | 17 +++ .../com/newrelic/agent/stats/StatsEngine.java | 2 + .../newrelic/agent/stats/StatsEngineImpl.java | 8 ++ .../com/newrelic/agent/stats/StatsWorks.java | 4 + .../agent/transport/DataSenderImpl.java | 43 +++++- 9 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStats.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java index 32bd3a2772..783a8132d5 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java @@ -389,6 +389,12 @@ public class MetricNames { //Supportability metric indicating that the payload was too large public static final String SUPPORTABILITY_PAYLOAD_SIZE_EXCEEDS_MAX = "Supportability/Agent/Collector/MaxPayloadSizeLimit/{0}"; + // Supportability metrics for uncompressed data payloads used to measure usage + // {0} = destination (Collector, OTLP, or InfiniteTracing). + public static final String SUPPORTABILITY_DATA_USAGE_DESTINATION_OUTPUT_BYTES = "Supportability/Java/{0}/Output/Bytes"; + // {0} = destination (Collector, OTLP, or InfiniteTracing). {1} = agent endpoint (connect, analytic_event_data, error_data, etc) + public static final String SUPPORTABILITY_DATA_USAGE_DESTINATION_ENDPOINT_OUTPUT_BYTES = "Supportability/Java/{0}/{1}/Output/Bytes"; + public static final String SUPPORTABILITY_AGENT_CONNECT_BACKOFF_ATTEMPTS = "Supportability/Agent/Collector/Connect/BackoffAttempts"; // expected errors diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStats.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStats.java new file mode 100644 index 0000000000..1d51f8da8c --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStats.java @@ -0,0 +1,44 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.stats; + +/** + * Used to generate metrics on data usage + */ +public interface DataUsageStats extends StatsBase { + + /** + * Record the uncompressed sizes of sent and received payloads in bytes for each agent endpoint. + * + * @param bytesSent uncompressed bytes sent to an agent endpoint + * @param bytesReceived uncompressed bytes received from an agent endpoint + */ + void recordDataUsage(long bytesSent, long bytesReceived); + + /** + * Get the count of the number of times the metric was set. + * + * @return count + */ + int getCount(); + + /** + * Get the amount of uncompressed bytes sent for a metric representing calls to an agent endpoint. + * + * @return bytes sent + */ + long getBytesSent(); + + /** + * Get the amount of uncompressed bytes received for a metric representing responses from an agent endpoint. + * + * @return bytes sent + */ + long getBytesReceived(); + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java new file mode 100644 index 0000000000..a7ec9487a8 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java @@ -0,0 +1,136 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.stats; + +import com.newrelic.agent.Agent; +import com.newrelic.agent.config.AgentConfigImpl; +import com.newrelic.api.agent.NewRelic; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; + +public class DataUsageStatsImpl implements DataUsageStats { + + private static final int UNUSED = 0; // min, max, sum of squares + private final AtomicInteger count = new AtomicInteger(0); // count + private final AtomicLong bytesSent = new AtomicLong(0); // total time + private final AtomicLong bytesReceived = new AtomicLong(0); // exclusive time + + protected DataUsageStatsImpl() { + super(); + } + +// // Used by the server mode +// public DataUsageStatsImpl(int count, int bytesSent, int bytesReceived) { +// super(); +// this.count = count; +// this.bytesSent = bytesSent; +// this.bytesReceived = bytesReceived; +// } + + /** + * Record the uncompressed sizes of sent and received payloads in bytes for each agent endpoint. + * + * @param bytesSent uncompressed bytes sent to an agent endpoint + * @param bytesReceived uncompressed bytes received from an agent endpoint + */ + @Override + public void recordDataUsage(long bytesSent, long bytesReceived) { + this.count.incrementAndGet(); + this.bytesSent.addAndGet(bytesSent); + this.bytesReceived.addAndGet(bytesReceived); + + if (NewRelic.getAgent().getConfig().getValue(AgentConfigImpl.METRIC_DEBUG, AgentConfigImpl.DEFAULT_METRIC_DEBUG)) { + if (this.count.get() < 0 || this.bytesSent.get() < 0 || this.bytesReceived.get() < 0) { + NewRelic.incrementCounter("Supportability/DataUsageStatsImpl/NegativeValue"); + Agent.LOG.log(Level.INFO, "Invalid count {0}, bytesSent {1}, or bytesReceived {2}", + this.count.get(), this.bytesSent.get(), this.bytesReceived.get()); + } + } + + } + + /** + * Get the count of the number of times the metric was set. + * + * @return count + */ + @Override + public int getCount() { + return count.get(); + } + + /** + * Get the amount of uncompressed bytes sent for a metric representing calls to an agent endpoint. + * + * @return bytes sent + */ + @Override + public long getBytesSent() { + return bytesSent.get(); + } + + /** + * Get the amount of uncompressed bytes received for a metric representing responses from an agent endpoint. + * + * @return bytes sent + */ + @Override + public long getBytesReceived() { + return bytesReceived.get(); + } + + @Override + public Object clone() throws CloneNotSupportedException { + DataUsageStatsImpl newStats = new DataUsageStatsImpl(); + newStats.count.set(count.get()); + newStats.bytesSent.set(bytesSent.get()); + newStats.bytesReceived.set(bytesReceived.get()); + return newStats; + } + + @Override + public String toString() { + return super.toString() + " [count=" + count.get() + ", bytesSent=" + bytesSent.get() + ", bytesReceived=" + bytesReceived.get() + "]"; + } + + @Override + public boolean hasData() { + return count.get() > 0 || bytesSent.get() > 0 || bytesReceived.get() > 0; + } + + @Override + public void reset() { + count.set(0); + bytesSent.set(0); + bytesReceived.set(0); + } + + @Override + public void writeJSONString(Writer writer) throws IOException { + List data; + // These map to traditional metric values of: count, total time, exclusive time, min, max, sum of squares + data = Arrays.asList(count.get(), bytesSent.get(), bytesReceived.get(), UNUSED, UNUSED, UNUSED); + org.json.simple.JSONArray.writeJSONString(data, writer); + } + + @Override + public void merge(StatsBase statsObj) { + if (statsObj instanceof DataUsageStatsImpl) { + DataUsageStatsImpl stats = (DataUsageStatsImpl) statsObj; + count.addAndGet(stats.count.get()); + bytesSent.addAndGet(stats.bytesSent.get()); + bytesReceived.addAndGet(stats.bytesReceived.get()); + } + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java new file mode 100644 index 0000000000..63bff00ce0 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java @@ -0,0 +1,33 @@ +/* + * + * * Copyright 2020 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.stats; + +import com.newrelic.agent.metric.MetricName; + +final class RecordDataUsageMetric implements StatsWork { + private final MetricName name; + private final long bytesSent; + private final long bytesReceived; + + public RecordDataUsageMetric(String name, long bytesSent, long bytesReceived) { + this.name = MetricName.create(name); + this.bytesSent = bytesSent; + this.bytesReceived = bytesReceived; + } + + @Override + public void doWork(StatsEngine statsEngine) { + statsEngine.getDataUsageStats(name).recordDataUsage(bytesSent, bytesReceived); + } + + @Override + public String getAppName() { + return null; + } + +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/SimpleStatsEngine.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/SimpleStatsEngine.java index c6531344dc..7f1beb27d9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/SimpleStatsEngine.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/SimpleStatsEngine.java @@ -106,6 +106,23 @@ public ApdexStats getApdexStats(String metricName) { } } + public DataUsageStats getDataUsageStats(String metricName) { + if (metricName == null) { + throw new RuntimeException("Cannot get a stat for a null metric"); + } + StatsBase s = stats.get(metricName); + if (s == null) { + s = new DataUsageStatsImpl(); + stats.put(metricName, s); + } + if (s instanceof DataUsageStats) { + return (DataUsageStats) s; + } else { + String msg = MessageFormat.format("The stats object for {0} is of type {1}", metricName, s.getClass().getName()); + throw new RuntimeException(msg); + } + } + public void mergeStats(SimpleStatsEngine other) { for (Entry entry : other.stats.entrySet()) { StatsBase ourStats = stats.get(entry.getKey()); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngine.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngine.java index b2d2e70ec5..6eab9f6280 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngine.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngine.java @@ -42,6 +42,8 @@ public interface StatsEngine { ApdexStats getApdexStats(MetricName metric); + DataUsageStats getDataUsageStats(MetricName metric); + /** * This is now only used by tests. */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngineImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngineImpl.java index 13f31502dc..9d138cc6eb 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngineImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsEngineImpl.java @@ -114,6 +114,14 @@ public ApdexStats getApdexStats(MetricName metricName) { return getStatsEngine(metricName).getApdexStats(metricName.getName()); } + @Override + public DataUsageStats getDataUsageStats(MetricName metricName) { + if (metricName == null) { + throw new RuntimeException("Cannot get a stat for a null metric"); + } + return getStatsEngine(metricName).getDataUsageStats(metricName.getName()); + } + @Override public List getMetricNames() { List result = new ArrayList<>(getSize()); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsWorks.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsWorks.java index 737d667f95..b99774a6b7 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsWorks.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/StatsWorks.java @@ -21,6 +21,10 @@ public static StatsWork getRecordMetricWork(String name, float value) { return new RecordMetric(name, value); } + public static StatsWork getRecordDataUsageMetricWork(String name, long bytesSent, long bytesReceived) { + return new RecordDataUsageMetric(name, bytesSent, bytesReceived); + } + public static StatsWork getRecordResponseTimeWork(String name, long millis) { return new RecordResponseTimeMetric(millis, name, TimeUnit.MILLISECONDS); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 6d5ade4483..81665effaa 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -90,7 +90,11 @@ public class DataSenderImpl implements DataSender { private static final String METADATA_PREFIX = "NEW_RELIC_METADATA_"; // the block of env vars we send up to rpm private static final String ENV_METADATA = "metadata"; - private static final int DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 1000000; + private static final int DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 1_000_000; + + private static final String COLLECTOR = "Collector"; + private static final String OTLP = "OTLP"; + private static final String INFINITE_TRACING = "InfiniteTracing"; // As of P17 these are the only agent endpoints that actually contain data in the response payload for a successful request private static final Set METHODS_WITH_RESPONSE_BODY = ImmutableSet.of( @@ -589,9 +593,10 @@ private ReadResult connectAndSend(String host, String method, String encoding, S ReadResult result = httpClientWrapper.execute(request, new TimingEventHandler(method, ServiceFactory.getStatsService())); + String payloadJsonSent = DataSenderWriter.toJSONString(params); + if (auditMode && methodShouldBeAudited(method)) { - String msg = MessageFormat.format("Sent JSON({0}) to: {1}, with payload: {2}", method, url, - DataSenderWriter.toJSONString(params)); + String msg = MessageFormat.format("Sent JSON({0}) to: {1}, with payload: {2}", method, url, payloadJsonSent); logger.info(msg); } @@ -603,11 +608,17 @@ private ReadResult connectAndSend(String host, String method, String encoding, S throwExceptionFromStatusCode(method, result, data, request); } + String payloadJsonReceived = result.getResponseBody(); + // received successful 2xx response if (auditMode && methodShouldBeAudited(method)) { - logger.info(MessageFormat.format("Received JSON({0}): {1}", method, result.getResponseBody())); + logger.info(MessageFormat.format("Received JSON({0}): {1}", method, payloadJsonReceived)); } + // TODO some logic to determine the correct destination, maybe PROTOCOL? + // Is this the right place to call this method??? Or should it be before the exception logic? + recordDataUsageMetrics(COLLECTOR, method, payloadJsonSent, payloadJsonReceived); + if (dataSenderListener != null) { dataSenderListener.dataSent(method, encoding, uri, data); } @@ -615,6 +626,30 @@ private ReadResult connectAndSend(String host, String method, String encoding, S return result; } + /** + * Record metrics tracking amount of bytes sent and received for each agent endpoint payload + * + * @param destination data destination (COLLECTOR, OTLP, INFINITE_TRACING) + * @param method method for the agent endpoint + * @param payloadJsonSent JSON String of the payload that was sent + * @param payloadJsonReceived JSON String of the payload that was received + */ + private void recordDataUsageMetrics(String destination, String method, String payloadJsonSent, String payloadJsonReceived) { + int payloadBytesSent = payloadJsonSent.getBytes().length; + int payloadBytesReceived = payloadJsonReceived.getBytes().length; + + // TODO figure out how to tell the destination of the data Collector, OTLP, or InfiniteTracing + ServiceFactory.getStatsService().doStatsWork( + StatsWorks.getRecordDataUsageMetricWork( + MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_OUTPUT_BYTES, destination), + payloadBytesSent, payloadBytesReceived)); + + ServiceFactory.getStatsService().doStatsWork( + StatsWorks.getRecordDataUsageMetricWork( + MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_ENDPOINT_OUTPUT_BYTES, COLLECTOR, method), + payloadBytesSent, payloadBytesReceived)); + } + private void throwExceptionFromStatusCode(String method, ReadResult result, byte[] data, HttpClientWrapper.Request request) throws HttpError, LicenseException, ForceRestartException, ForceDisconnectException { // Comply with spec and send supportability metric only for error responses From b3247e005218fc157f4f773d9e7906234d84f016 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 8 Feb 2022 10:06:57 -0800 Subject: [PATCH 28/96] Refactoring --- .../main/java/com/newrelic/SpanEventSender.java | 3 +++ .../agent/transport/DataSenderImpl.java | 17 ++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java b/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java index 9a4128d21c..e2113bbb95 100644 --- a/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java +++ b/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java @@ -17,6 +17,8 @@ class SpanEventSender implements Runnable { private final BlockingQueue queue; private final MetricAggregator aggregator; private final ChannelManager channelManager; + // Destination for agent data + private static final String INFINITE_TRACING = "InfiniteTracing"; SpanEventSender(InfiniteTracingConfig config, BlockingQueue queue, MetricAggregator aggregator, ChannelManager channelManager) { this.logger = config.getLogger(); @@ -97,6 +99,7 @@ void writeToObserver(ClientCallStreamObserver observer, V1.Span span) { logger.log(Level.SEVERE, t, "Unable to send span."); throw t; } + // TODO record data usage metrics here for the INFINITE_TRACING destination aggregator.incrementCounter("Supportability/InfiniteTracing/Span/Sent"); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 81665effaa..fb3eaee27b 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -92,9 +92,9 @@ public class DataSenderImpl implements DataSender { private static final String ENV_METADATA = "metadata"; private static final int DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 1_000_000; + // Destinations for agent data private static final String COLLECTOR = "Collector"; - private static final String OTLP = "OTLP"; - private static final String INFINITE_TRACING = "InfiniteTracing"; + private static final String OTLP = "OTLP"; // Not currently supported by Java agent // As of P17 these are the only agent endpoints that actually contain data in the response payload for a successful request private static final Set METHODS_WITH_RESPONSE_BODY = ImmutableSet.of( @@ -615,9 +615,7 @@ private ReadResult connectAndSend(String host, String method, String encoding, S logger.info(MessageFormat.format("Received JSON({0}): {1}", method, payloadJsonReceived)); } - // TODO some logic to determine the correct destination, maybe PROTOCOL? - // Is this the right place to call this method??? Or should it be before the exception logic? - recordDataUsageMetrics(COLLECTOR, method, payloadJsonSent, payloadJsonReceived); + recordDataUsageMetrics(method, payloadJsonSent, payloadJsonReceived); if (dataSenderListener != null) { dataSenderListener.dataSent(method, encoding, uri, data); @@ -629,19 +627,20 @@ private ReadResult connectAndSend(String host, String method, String encoding, S /** * Record metrics tracking amount of bytes sent and received for each agent endpoint payload * - * @param destination data destination (COLLECTOR, OTLP, INFINITE_TRACING) * @param method method for the agent endpoint * @param payloadJsonSent JSON String of the payload that was sent * @param payloadJsonReceived JSON String of the payload that was received */ - private void recordDataUsageMetrics(String destination, String method, String payloadJsonSent, String payloadJsonReceived) { + private void recordDataUsageMetrics(String method, String payloadJsonSent, String payloadJsonReceived) { int payloadBytesSent = payloadJsonSent.getBytes().length; int payloadBytesReceived = payloadJsonReceived.getBytes().length; - // TODO figure out how to tell the destination of the data Collector, OTLP, or InfiniteTracing + // COLLECTOR is always the destination for data reported via DataSenderImpl. + // OTLP as a destination is not currently supported by the Java agent. + // INFINITE_TRACING destined usage data is sent via SpanEventSender. ServiceFactory.getStatsService().doStatsWork( StatsWorks.getRecordDataUsageMetricWork( - MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_OUTPUT_BYTES, destination), + MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_OUTPUT_BYTES, COLLECTOR), payloadBytesSent, payloadBytesReceived)); ServiceFactory.getStatsService().doStatsWork( From d7cc8a419d92b55ea66feb1f169cb658dbcb7212 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 9 Feb 2022 09:39:54 -0800 Subject: [PATCH 29/96] Add tests --- .../com/newrelic/agent/DummyTransaction.java | 64 +++++++++++++++++++ .../agent/stats/DataUsageStatsImpl.java | 8 --- .../agent/stats/RecordDataUsageMetric.java | 13 +++- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java index 2847db673e..1c8f04b32f 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java @@ -26,6 +26,8 @@ import com.newrelic.agent.stats.AbstractMetricAggregator; import com.newrelic.agent.stats.ApdexStats; import com.newrelic.agent.stats.ApdexStatsImpl; +import com.newrelic.agent.stats.DataUsageStats; +import com.newrelic.agent.stats.DataUsageStatsImpl; import com.newrelic.agent.stats.ResponseTimeStats; import com.newrelic.agent.stats.ResponseTimeStatsImpl; import com.newrelic.agent.stats.SimpleStatsEngine; @@ -69,6 +71,10 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +/** + * DummyTransaction is a lightweight Transaction that gets returned when the agent's circuit + * breaker has been tripped in order to minimize the agent's effects on a JVM running near its limits. + */ public class DummyTransaction extends Transaction { private final String guid; @@ -851,6 +857,7 @@ static final class DummySimpleStatsEngine extends SimpleStatsEngine { static final DummyStats stat = new DummyStats(); static final DummyResponseTimeStats responseTimeStat = new DummyResponseTimeStats(); static final DummyApdexStat apdexStat = new DummyApdexStat(); + static final DummyDataUsageStats dataUsageStats = new DummyDataUsageStats(); @Override public Map getStatsMap() { @@ -876,6 +883,11 @@ public ApdexStats getApdexStats(String metricName) { return apdexStat; } + @Override + public DataUsageStats getDataUsageStats(String metricName) { + return dataUsageStats; + } + @Override public void mergeStats(SimpleStatsEngine other) { } @@ -1121,6 +1133,58 @@ public void merge(StatsBase statsObj) { } } + static final class DummyDataUsageStats extends DataUsageStatsImpl { + DummyDataUsageStats() { + super(); + } + + @Override + public Object clone() throws CloneNotSupportedException { + return this; + } + + @Override + public String toString() { + return ""; + } + + @Override + public void recordDataUsage(long bytesSent, long bytesReceived) { + } + + @Override + public int getCount() { + return 0; + } + + @Override + public long getBytesSent() { + return 0; + } + + @Override + public long getBytesReceived() { + return 0; + } + + @Override + public boolean hasData() { + return false; + } + + @Override + public void reset() { + } + + @Override + public void writeJSONString(Writer writer) throws IOException { + } + + @Override + public void merge(StatsBase statsObj) { + } + } + static final class DummyTransactionCache extends TransactionCache { public static final TransactionCache INSTANCE = new DummyTransactionCache(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java index a7ec9487a8..e7466e77f4 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/DataUsageStatsImpl.java @@ -30,14 +30,6 @@ protected DataUsageStatsImpl() { super(); } -// // Used by the server mode -// public DataUsageStatsImpl(int count, int bytesSent, int bytesReceived) { -// super(); -// this.count = count; -// this.bytesSent = bytesSent; -// this.bytesReceived = bytesReceived; -// } - /** * Record the uncompressed sizes of sent and received payloads in bytes for each agent endpoint. * diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java index 63bff00ce0..5ef3132fec 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java @@ -9,7 +9,7 @@ import com.newrelic.agent.metric.MetricName; -final class RecordDataUsageMetric implements StatsWork { +public final class RecordDataUsageMetric implements StatsWork { private final MetricName name; private final long bytesSent; private final long bytesReceived; @@ -30,4 +30,15 @@ public String getAppName() { return null; } + public String getName() { + return name.getName(); + } + + public long getBytesSent() { + return bytesSent; + } + + public long getBytesReceived() { + return bytesReceived; + } } From 0a9bb8bd15d4d9b53f8310fee6db368e7536bbab Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 9 Feb 2022 09:53:20 -0800 Subject: [PATCH 30/96] Add tests --- .../newrelic/agent/stats/StatsEngineTest.java | 85 ++++++++++++++++ .../agent/stats/StatsServiceTest.java | 37 ++++++- .../agent/transport/DataSenderImplTest.java | 96 ++++++++++++++++++- 3 files changed, 211 insertions(+), 7 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsEngineTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsEngineTest.java index f747c88ea5..d9ef130422 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsEngineTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsEngineTest.java @@ -432,6 +432,27 @@ public void getMetricNames() { Assert.assertEquals(0, data.size()); } + @Test + public void getMetricNamesFromDataUsageStats() { + StatsEngineImpl statsEngine = new StatsEngineImpl(); + DataUsageStats stats = statsEngine.getDataUsageStats(MetricName.create("Test")); + stats.recordDataUsage(100, 5); + stats.recordDataUsage(300, 10); + + List data = statsEngine.getMetricNames(); + Assert.assertEquals(1, data.size()); + Assert.assertEquals(1, statsEngine.getSize()); + Assert.assertEquals("Test", data.get(0).getName()); + DataUsageStats harvestedStats = statsEngine.getDataUsageStats(data.get(0)); + Assert.assertEquals(400, harvestedStats.getBytesSent()); + Assert.assertEquals(15, harvestedStats.getBytesReceived()); + + statsEngine.clear(); + data.clear(); + data = statsEngine.getMetricNames(); + Assert.assertEquals(0, data.size()); + } + @Test public void getMetricData() { StatsEngineImpl statsEngine = new StatsEngineImpl(); @@ -449,6 +470,23 @@ public void getMetricData() { Assert.assertEquals(2, ((CountStats) data.get(0).getStats()).getCallCount()); } + @Test + public void getMetricDataFromDataUsageStats() { + StatsEngineImpl statsEngine = new StatsEngineImpl(); + DataUsageStats stats = statsEngine.getDataUsageStats(MetricName.create("Test")); + stats.recordDataUsage(100, 5); + stats.recordDataUsage(300, 10); + + MockNormalizer metricNormalizer = new MockNormalizer(); + List data = statsEngine.getMetricData(metricNormalizer); + Assert.assertEquals(1, data.size()); + Assert.assertEquals(1, statsEngine.getSize()); + Assert.assertEquals("Test", data.get(0).getMetricName().getName()); + Assert.assertNull(data.get(0).getMetricId()); + Assert.assertEquals(400, ((DataUsageStats) data.get(0).getStats()).getBytesSent()); + Assert.assertEquals(15, ((DataUsageStats) data.get(0).getStats()).getBytesReceived()); + } + @Test public void getMetricDataNormalize() { StatsEngineImpl statsEngine = new StatsEngineImpl(); @@ -469,6 +507,26 @@ public void getMetricDataNormalize() { Assert.assertEquals(2, ((CountStats) data.get(0).getStats()).getCallCount()); } + @Test + public void getMetricDataNormalizeFromDataUsageStats() { + StatsEngineImpl statsEngine = new StatsEngineImpl(); + DataUsageStats stats = statsEngine.getDataUsageStats(MetricName.create("Test")); + stats.recordDataUsage(100, 5); + stats.recordDataUsage(300, 10); + + MockNormalizer metricNormalizer = new MockNormalizer(); + Map normalizationResults = new HashMap<>(); + normalizationResults.put("Test", "Test2"); + metricNormalizer.setNormalizationResults(normalizationResults); + List data = statsEngine.getMetricData(metricNormalizer); + Assert.assertEquals(1, data.size()); + Assert.assertEquals(1, statsEngine.getSize()); + Assert.assertEquals("Test2", data.get(0).getMetricName().getName()); + Assert.assertNull(data.get(0).getMetricId()); + Assert.assertEquals(400, ((DataUsageStats) data.get(0).getStats()).getBytesSent()); + Assert.assertEquals(15, ((DataUsageStats) data.get(0).getStats()).getBytesReceived()); + } + @Test public void merge() { StatsEngineImpl statsEngine = new StatsEngineImpl(); @@ -493,4 +551,31 @@ public void merge() { Assert.assertEquals(100f, stats3.getTotal(), 0); } + @Test + public void mergeDataUsageStats() { + StatsEngineImpl statsEngine = new StatsEngineImpl(); + DataUsageStats stats1 = statsEngine.getDataUsageStats(MetricName.create("Test1")); + stats1.recordDataUsage(100, 5); + DataUsageStats stats2 = statsEngine.getDataUsageStats(MetricName.create("Test2")); + stats2.recordDataUsage(300, 10); + + StatsEngineImpl statsEngine2 = new StatsEngineImpl(); + DataUsageStats stats3 = statsEngine.getDataUsageStats(MetricName.create("Test3")); + stats3.recordDataUsage(200, 20); + DataUsageStats stats4 = statsEngine.getDataUsageStats(MetricName.create("Test2")); + stats4.recordDataUsage(400, 50); + + statsEngine.mergeStats(statsEngine2); + Assert.assertEquals(3, statsEngine.getSize()); + Assert.assertEquals(1, stats1.getCount()); + Assert.assertEquals(2, stats2.getCount()); + Assert.assertEquals(1, stats3.getCount()); + Assert.assertEquals(100, stats1.getBytesSent()); + Assert.assertEquals(700, stats2.getBytesSent()); + Assert.assertEquals(200, stats3.getBytesSent()); + Assert.assertEquals(5, stats1.getBytesReceived()); + Assert.assertEquals(60, stats2.getBytesReceived()); + Assert.assertEquals(20, stats3.getBytesReceived()); + } + } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsServiceTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsServiceTest.java index 5b1902ccb6..4bfedada3c 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsServiceTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/stats/StatsServiceTest.java @@ -13,6 +13,7 @@ import com.newrelic.agent.config.AgentConfigImpl; import com.newrelic.agent.config.ConfigService; import com.newrelic.agent.config.ConfigServiceFactory; +import com.newrelic.agent.metric.MetricName; import com.newrelic.agent.service.ServiceFactory; import com.newrelic.agent.service.ServiceManager; import org.junit.After; @@ -71,19 +72,51 @@ private MockServiceManager createServiceManager(Map map) throws @Test public void doStatsWork() throws Exception { String appName = serviceManager.getConfigService().getDefaultAgentConfig().getApplicationName(); - StatsEngineImpl statsEngine = new StatsEngineImpl(); StatsService statsService = serviceManager.getStatsService(); + + // RecordMetric count 1 + StatsEngineImpl statsEngine = new StatsEngineImpl(); Stats stats1 = statsEngine.getStats("Test1"); stats1.recordDataPoint(100f); statsService.doStatsWork(new MergeStatsWork(appName, statsEngine)); + + // RecordMetric count 2 statsEngine = new StatsEngineImpl(); stats1 = statsEngine.getStats("Test1"); stats1.recordDataPoint(200f); statsService.doStatsWork(new MergeStatsWork(appName, statsEngine)); + + // RecordMetric count 3 statsService.doStatsWork(new RecordMetric("Test1", 300f)); + + // RecordDataUsageMetric count 1 + statsEngine = new StatsEngineImpl(); + DataUsageStats dataUsageStats = statsEngine.getDataUsageStats(MetricName.create("Test2")); + dataUsageStats.recordDataUsage(5000, 25); + statsService.doStatsWork(new MergeStatsWork(appName, statsEngine)); + + // RecordDataUsageMetric count 2 + statsEngine = new StatsEngineImpl(); + dataUsageStats = statsEngine.getDataUsageStats(MetricName.create("Test2")); + dataUsageStats.recordDataUsage(1000, 10); + statsService.doStatsWork(new MergeStatsWork(appName, statsEngine)); + + // RecordDataUsageMetric count 3 + statsService.doStatsWork(new RecordDataUsageMetric("Test2", 100, 5)); + StatsEngine harvestStatsEngine = statsService.getStatsEngineForHarvest(appName); - Assert.assertEquals(1, harvestStatsEngine.getSize()); + + // Number of unique metrics (Test1 and Test2) + Assert.assertEquals(2, harvestStatsEngine.getSize()); + + // Test1 totals + Assert.assertEquals(3, harvestStatsEngine.getStats("Test1").getCallCount()); Assert.assertEquals(600f, harvestStatsEngine.getStats("Test1").getTotal(), 0); + + // Test2 totals + Assert.assertEquals(3, harvestStatsEngine.getDataUsageStats(MetricName.create("Test2")).getCount()); + Assert.assertEquals(6100, harvestStatsEngine.getDataUsageStats(MetricName.create("Test2")).getBytesSent()); + Assert.assertEquals(40, harvestStatsEngine.getDataUsageStats(MetricName.create("Test2")).getBytesReceived()); } @Test diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index ad1f55cf01..10cf8ca92d 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -27,6 +27,7 @@ import com.newrelic.agent.service.analytics.TransactionEvent; import com.newrelic.agent.service.analytics.TransactionEventBuilder; import com.newrelic.agent.stats.IncrementCounter; +import com.newrelic.agent.stats.RecordDataUsageMetric; import com.newrelic.agent.stats.StatsImpl; import com.newrelic.agent.stats.StatsService; import org.hamcrest.CoreMatchers; @@ -239,6 +240,45 @@ public void dataReceived(String method, String encoding, String uri, Map r assertTrue("expected dataSent to be called!", dataSentCalled.get()); } + @Test + public void testDataUsageSupportability() throws Exception { + AgentConfig config = AgentConfigImpl.createAgentConfig(configMap()); + HttpClientWrapper wrapperEmptyReturn = getHttpClientWrapper(ReadResult.create( + HttpResponseCode.OK, + "", + null)); + + DataSenderImpl target = new DataSenderImpl(config, wrapperEmptyReturn, null, logger, ServiceFactory.getConfigService()); + + target.setAgentRunId("agent run id"); + + List metricData = createMetricData(5); + target.sendMetricData(System.currentTimeMillis() - 5000, System.currentTimeMillis(), metricData); + + // Expected payload sent after adding 5 metrics and agent metadata + final String expectedSentPayload = "[\"agent run id\",1644424673,1644424678," + + "[[0,[1,1.0,1.0,1.0,1.0,1.0]]," + + "[1,[1,1.0,1.0,1.0,1.0,1.0]]," + + "[2,[1,1.0,1.0,1.0,1.0,1.0]]," + + "[3,[1,1.0,1.0,1.0,1.0,1.0]]," + + "[4,[1,1.0,1.0,1.0,1.0,1.0]]]]"; + int expectedSentPayloadSizeInBytes = expectedSentPayload.getBytes().length; + + // Expected payload received is empty + final String expectedReceivedPayload = ""; + int expectedReceivedPayloadSizeInBytes = expectedReceivedPayload.getBytes().length; + + String collectorOutputBytesMetric = MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_OUTPUT_BYTES, "Collector"); + String collectorEndpointOutputBytesMetric = MessageFormat.format(MetricNames.SUPPORTABILITY_DATA_USAGE_DESTINATION_ENDPOINT_OUTPUT_BYTES, "Collector", "metric_data"); + + assertMetricWasRecorded(MessageFormat.format(MetricNames.SUPPORTABILITY_HTTP_CODE, HttpResponseCode.OK)); + assertMetricWasRecorded(collectorOutputBytesMetric); + assertMetricWasRecorded(collectorEndpointOutputBytesMetric); + + assertDataUsageMetricValues(collectorOutputBytesMetric, expectedSentPayloadSizeInBytes, expectedReceivedPayloadSizeInBytes); + assertDataUsageMetricValues(collectorEndpointOutputBytesMetric, expectedSentPayloadSizeInBytes, expectedReceivedPayloadSizeInBytes); + } + @Test public void testSuccessSupportability() throws Exception { AgentConfig config = AgentConfigImpl.createAgentConfig(configMap()); @@ -429,16 +469,54 @@ public void testReturnsParsedAgentCommands() throws Exception { assertEquals(4L, result.get(0).get(1)); } + /** + * Verify that a given metric was created + * + * @param expectedMetricName name of metric to verify + */ private void assertMetricWasRecorded(String expectedMetricName) { boolean found = false; MockingDetails output = Mockito.mockingDetails(mockStatsService); - for(Invocation invocation: output.getInvocations()) { - found = found || ( - invocation.getMethod().getName().equals("doStatsWork") - && invocation.getArgument(0).getName().equals(expectedMetricName) - ); + for (Invocation invocation: output.getInvocations()) { + if (found) { + break; + } + String methodName = invocation.getMethod().getName(); + Object rawArgument = invocation.getRawArguments()[0]; + if (rawArgument instanceof IncrementCounter) { + String metricName = invocation.getArgument(0).getName(); + found = methodName.equals("doStatsWork") && metricName.equals(expectedMetricName); + } else if (rawArgument instanceof RecordDataUsageMetric) { + String metricName = invocation.getArgument(0).getName(); + found = methodName.equals("doStatsWork") && metricName.equals(expectedMetricName); + } } + assertTrue("Could not find metric: " + expectedMetricName, found); + } + /** + * Verify the sent/received payload sizes recorded by a given RecordDataUsageMetric + * + * @param expectedMetricName name of metric to verify + * @param expectedBytesSent expected size of sent payload in bytes + * @param expectedBytesReceived expected size of received payload in bytes + */ + private void assertDataUsageMetricValues(String expectedMetricName, int expectedBytesSent, int expectedBytesReceived) { + boolean found = false; + MockingDetails output = Mockito.mockingDetails(mockStatsService); + for (Invocation invocation: output.getInvocations()) { + String methodName = invocation.getMethod().getName(); + Object rawArgument = invocation.getRawArguments()[0]; + if (rawArgument instanceof RecordDataUsageMetric) { + String metricName = invocation.getArgument(0).getName(); + found = methodName.equals("doStatsWork") && metricName.equals(expectedMetricName); + if (found) { + assertEquals(expectedBytesSent, ((RecordDataUsageMetric) rawArgument).getBytesSent()); + assertEquals(expectedBytesReceived, ((RecordDataUsageMetric) rawArgument).getBytesReceived()); + break; + } + } + } assertTrue("Could not find metric: " + expectedMetricName, found); } @@ -582,6 +660,14 @@ private List createMetricData(int metrics) { } return metricData; } +// +// private List createMetricData(int metrics) { +// List metricData = new ArrayList<>(); +// for (int i = 0; i < metrics; i++) { +// metricData.add(MetricData.create(MetricName.create(String.valueOf(i)), i, new StatsImpl(1, 1, 1, 1, 1))); +// } +// return metricData; +// } public Map configMap() { Map configMap = new HashMap<>(); From d5af4123389914e9d0c5ec67af58083e03841b91 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 9 Feb 2022 10:17:48 -0800 Subject: [PATCH 31/96] Some cleanup --- .../com/newrelic/agent/stats/RecordDataUsageMetric.java | 2 +- .../com/newrelic/agent/transport/DataSenderImplTest.java | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java index 5ef3132fec..f9c289c769 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/stats/RecordDataUsageMetric.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index 10cf8ca92d..50c8477b4e 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -660,14 +660,6 @@ private List createMetricData(int metrics) { } return metricData; } -// -// private List createMetricData(int metrics) { -// List metricData = new ArrayList<>(); -// for (int i = 0; i < metrics; i++) { -// metricData.add(MetricData.create(MetricName.create(String.valueOf(i)), i, new StatsImpl(1, 1, 1, 1, 1))); -// } -// return metricData; -// } public Map configMap() { Map configMap = new HashMap<>(); From 6951454abcaa684219bdec58d483417971ce0ec2 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Feb 2022 09:51:17 -0800 Subject: [PATCH 32/96] Update AgentUtil to handle config set by sys prop or env var --- .../instrumentation/log4j1/AgentUtil.java | 28 ++++++++++++++++--- .../instrumentation/log4j2/AgentUtil.java | 28 ++++++++++++++++--- .../com/nr/instrumentation/jul/AgentUtil.java | 28 ++++++++++++++++--- .../logbackclassic12/AgentUtil.java | 28 ++++++++++++++++--- 4 files changed, 96 insertions(+), 16 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 10f786b8f0..36ad77b7bb 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -57,18 +57,38 @@ public static Map getLinkingMetadataAsMap() { } public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 1dff2b98a8..c5ac5d440e 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -60,18 +60,38 @@ public static Map getLinkingMetadataAsMap() { } public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } } diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index f9a6fdc05b..5c200475ac 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -45,18 +45,38 @@ public static Map getLinkingMetadataAsMap() { } public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 1b99685384..e033641f6d 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -50,18 +50,38 @@ public static String getLinkingMetadataAsString() { } public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + // Config value is parsed as a String if it was set by system property or environment variable + if (configValue instanceof String) { + return Boolean.parseBoolean((String) configValue); + } + return (Boolean) configValue; } } From 6d784c07fcdcb997b2dfb0ad25596539ac844ed1 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Feb 2022 13:58:04 -0800 Subject: [PATCH 33/96] Remove plugin.type attribute --- .../main/java/com/newrelic/agent/transport/DataSenderImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index fb3eaee27b..8108d968ac 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -375,7 +375,6 @@ private void sendLogEventsForReserv } JSONObject commonAttributes = new JSONObject(); - commonAttributes.put("plugin.type", "nr-java-agent"); // TODO do we set this? or does backend service? // build attributes object JSONObject attributes = new JSONObject(); From b98cda27ab2352bb2be4e1f036be74feb9eaaa65 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Feb 2022 20:06:56 -0800 Subject: [PATCH 34/96] Add Logging supportability metrics --- .../java/com/newrelic/agent/MetricNames.java | 12 +++- .../service/logging/LogSenderServiceImpl.java | 57 +++++++++++++++---- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java index 783a8132d5..5427f89fc0 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/MetricNames.java @@ -218,8 +218,16 @@ public class MetricNames { public static final String SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SENT = "Supportability/Events/Customer/Sent"; public static final String SUPPORTABILITY_INSIGHTS_SERVICE_CUSTOMER_SEEN = "Supportability/Events/Customer/Seen"; - public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SENT = "Supportability/LogEvents/Customer/Sent"; - public static final String SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SEEN = "Supportability/LogEvents/Customer/Seen"; + public static final String SUPPORTABILITY_LOGGING_FORWARDING_SENT = "Supportability/Logging/Forwarding/Sent"; + public static final String SUPPORTABILITY_LOGGING_FORWARDING_SEEN = "Supportability/Logging/Forwarding/Seen"; + public static final String LOGGING_FORWARDING_DROPPED = "Logging/Forwarding/Dropped"; + + public static final String SUPPORTABILITY_LOGGING_METRICS_JAVA_ENABLED = "Supportability/Logging/Metrics/Java/enabled"; + public static final String SUPPORTABILITY_LOGGING_METRICS_JAVA_DISABLED = "Supportability/Logging/Metrics/Java/disabled"; + public static final String SUPPORTABILITY_LOGGING_FORWARDING_JAVA_ENABLED = "Supportability/Logging/Forwarding/Java/enabled"; + public static final String SUPPORTABILITY_LOGGING_FORWARDING_JAVA_DISABLED = "Supportability/Logging/Forwarding/Java/disabled"; + public static final String SUPPORTABILITY_LOGGING_LOCAL_DECORATING_JAVA_ENABLED = "Supportability/Logging/LocalDecorating/Java/enabled"; + public static final String SUPPORTABILITY_LOGGING_LOCAL_DECORATING_JAVA_DISABLED = "Supportability/Logging/LocalDecorating/Java/disabled"; public static final String SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/ReportPeriod"; public static final String SUPPORTABILITY_ERROR_SERVICE_REPORT_PERIOD_IN_SECONDS = "Supportability/EventHarvest/ErrorEventData/ReportPeriod"; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index fdae92d2dc..159302a887 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -25,6 +25,7 @@ import com.newrelic.agent.service.ServiceFactory; import com.newrelic.agent.service.analytics.DistributedSamplingPriorityQueue; import com.newrelic.agent.stats.StatsEngine; +import com.newrelic.agent.stats.StatsService; import com.newrelic.agent.stats.StatsWork; import com.newrelic.agent.stats.TransactionStats; import com.newrelic.agent.tracing.DistributedTraceServiceImpl; @@ -47,8 +48,8 @@ import static com.newrelic.agent.model.LogEvent.LOG_EVENT_TYPE; public class LogSenderServiceImpl extends AbstractService implements LogSenderService { - // Whether the service as a whole is enabled. Disabling shuts down all log events for transactions. - private volatile boolean enabled; + // Whether the service as a whole is enabled. Disabling shuts down all log events. + private volatile boolean forwardingEnabled; // Key is the app name, value is if it is enabled - should be a limited number of names private final ConcurrentMap isEnabledForApp = new ConcurrentHashMap<>(); // Number of log events in the reservoir sampling buffer per-app. All apps get the same value. @@ -94,19 +95,45 @@ public void dispatcherTransactionCancelled(Transaction transaction) { public void configChanged(String appName, AgentConfig agentConfig) { // if the config has changed for the app, just remove it and regenerate enabled next transaction isEnabledForApp.remove(appName); - enabled = agentConfig.getApplicationLoggingConfig().isForwardingEnabled(); + forwardingEnabled = agentConfig.getApplicationLoggingConfig().isForwardingEnabled(); maxSamplesStored = agentConfig.getApplicationLoggingConfig().getMaxSamplesStored(); + + boolean metricsEnabled = agentConfig.getApplicationLoggingConfig().isMetricsEnabled(); + boolean localDecoratingEnabled = agentConfig.getApplicationLoggingConfig().isLocalDecoratingEnabled(); + recordApplicationLoggingSupportabilityMetrics(forwardingEnabled, metricsEnabled, localDecoratingEnabled); } }; + public void recordApplicationLoggingSupportabilityMetrics(boolean forwardingEnabled, boolean metricsEnabled, boolean localDecoratingEnabled) { + StatsService statsService = ServiceFactory.getServiceManager().getStatsService(); + + if (forwardingEnabled) { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_FORWARDING_JAVA_ENABLED); + } else { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_FORWARDING_JAVA_DISABLED); + } + + if (metricsEnabled) { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_METRICS_JAVA_ENABLED); + } else { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_METRICS_JAVA_DISABLED); + } + + if (localDecoratingEnabled) { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_LOCAL_DECORATING_JAVA_ENABLED); + } else { + statsService.getMetricAggregator().incrementCounter(MetricNames.SUPPORTABILITY_LOGGING_LOCAL_DECORATING_JAVA_DISABLED); + } + } + private final List harvestables = new ArrayList<>(); public LogSenderServiceImpl() { super(LogSenderServiceImpl.class.getSimpleName()); AgentConfig config = ServiceFactory.getConfigService().getDefaultAgentConfig(); maxSamplesStored = config.getApplicationLoggingConfig().getMaxSamplesStored(); - enabled = config.getApplicationLoggingConfig().isForwardingEnabled(); - isEnabledForApp.put(config.getApplicationName(), enabled); + forwardingEnabled = config.getApplicationLoggingConfig().isForwardingEnabled(); + isEnabledForApp.put(config.getApplicationName(), forwardingEnabled); } /** @@ -115,7 +142,7 @@ public LogSenderServiceImpl() { */ @Override public boolean isEnabled() { - return enabled; + return forwardingEnabled; } /** @@ -293,13 +320,13 @@ private void createAndStoreEvent(String appName, Map attributes) { * @return true if they are disabled, false if they are enabled */ private boolean logEventsDisabled() { - if (!enabled) { + if (!forwardingEnabled) { if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); } else { - Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); // FIXME update these logs if log_sending is not used + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. application_logging.forwarding not enabled.", LOG_EVENT_TYPE); } - Agent.LOG.log(Level.FINER, "Event of type {0} not collected. log_sending not enabled.", LOG_EVENT_TYPE); // FIXME update these logs if log_sending is not used + Agent.LOG.log(Level.FINER, "Event of type {0} not collected. application_logging.forwarding not enabled.", LOG_EVENT_TYPE); return true; // LogEvents are disabled } return false; // LogEvents are enabled @@ -402,10 +429,18 @@ public String getEventHarvestLimitMetric() { private void recordSupportabilityMetrics(StatsEngine statsEngine, long durationInNanoseconds, DistributedSamplingPriorityQueue reservoir) { - statsEngine.getStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SENT) + statsEngine.getStats(MetricNames.SUPPORTABILITY_LOGGING_FORWARDING_SENT) .incrementCallCount(reservoir.size()); - statsEngine.getStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_CUSTOMER_SEEN) + statsEngine.getStats(MetricNames.SUPPORTABILITY_LOGGING_FORWARDING_SEEN) .incrementCallCount(reservoir.getNumberOfTries()); + + int droppedLogEvents = reservoir.getNumberOfTries() - reservoir.size(); + if (droppedLogEvents >= 0) { + statsEngine.getStats(MetricNames.LOGGING_FORWARDING_DROPPED).incrementCallCount(droppedLogEvents); + } else { + Agent.LOG.log(Level.FINE, "Invalid dropped log events value of {0}. This must be a non-negative value.", droppedLogEvents); + } + statsEngine.getResponseTimeStats(MetricNames.SUPPORTABILITY_LOG_SENDER_SERVICE_EVENT_HARVEST_TRANSMIT) .recordResponseTime(durationInNanoseconds, TimeUnit.NANOSECONDS); } From bbb3be7d0790aefc1b0bf538eda941b845687c12 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Feb 2022 19:28:03 -0800 Subject: [PATCH 35/96] Simplify instrumentation modules --- .../instrumentation/log4j1/AgentUtil.java | 107 ++++++++------- .../instrumentation/log4j1/ElementName.java | 18 --- .../instrumentation/log4j1/ExceptionUtil.java | 35 ----- .../log4j/Category_Instrumentation.java | 5 +- .../instrumentation/log4j2/AgentUtil.java | 125 ++++++++++++------ .../instrumentation/log4j2/ElementName.java | 18 --- .../instrumentation/log4j2/ExceptionUtil.java | 35 ----- .../config/LoggerConfig_Instrumentation.java | 6 +- .../StringBuilderEncoder_Instrumentation.java | 4 +- .../com/nr/instrumentation/jul/AgentUtil.java | 97 +++++++++----- .../nr/instrumentation/jul/ElementName.java | 18 --- .../nr/instrumentation/jul/ExceptionUtil.java | 35 ----- .../util/logging/Logger_Instrumentation.java | 8 +- .../spi/LoggingEvent_Instrumentation.java | 10 +- .../logbackclassic12/AgentUtil.java | 112 +++++++++++----- .../logbackclassic12/ElementName.java | 18 --- .../logbackclassic12/ExceptionUtil.java | 35 ----- .../newrelic/agent/config/ConfigConstant.java | 2 +- 18 files changed, 310 insertions(+), 378 deletions(-) delete mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java delete mode 100644 instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java delete mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java delete mode 100644 instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java delete mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java delete mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java delete mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java delete mode 100644 instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 36ad77b7bb..bb98da3ff2 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -1,61 +1,63 @@ package com.nr.agent.instrumentation.log4j1; -import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; +import java.util.Collections; import java.util.HashMap; import java.util.Map; - -import static com.nr.agent.instrumentation.log4j1.ElementName.CLASS_NAME; -import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_CLASS; -import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_MESSAGE; -import static com.nr.agent.instrumentation.log4j1.ElementName.ERROR_STACK; -import static com.nr.agent.instrumentation.log4j1.ElementName.LOGGER_NAME; -import static com.nr.agent.instrumentation.log4j1.ElementName.LOG_LEVEL; -import static com.nr.agent.instrumentation.log4j1.ElementName.MESSAGE; -import static com.nr.agent.instrumentation.log4j1.ElementName.THROWABLE; -import static com.nr.agent.instrumentation.log4j1.ElementName.TIMESTAMP; +import java.util.Set; public class AgentUtil { + // Log message attributes + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String LOG_LEVEL = "log.level"; + public static final String UNKNOWN = "UNKNOWN"; + // Linking metadata attributes to filter out + private static final String ENTITY_TYPE = "entity.type"; + private static final String ENTITY_NAME = "entity.name"; - // TODO figure out how to recordNewRelicLogEvent for Log4j1 -// public static void recordNewRelicLogEvent(LogEvent event) { -// final String EMPTY_STRING = ""; -// -// if (event != null) { -// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); -// -// Message message = event.getMessage(); -// logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); -// logEventMap.put(TIMESTAMP, event.getTimeMillis()); -// -// Level level = event.getLevel(); -// logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); -// logEventMap.put(LOGGER_NAME, event.getLoggerName()); -// logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); -// -// Throwable throwable = event.getThrown(); -// if (throwable != null) { -// logEventMap.put(THROWABLE, throwable.toString()); -// logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); -// logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); -// logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); -// } -// -// AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); -// } -// } - - public static String getLinkingMetadataAsString() { - // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string - return NewRelic.getAgent().getLinkingMetadata().toString(); + /** + * Gets a String representing the agent linking metadata after filtering + * out entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered String of agent linking metadata + */ + public static String getFilteredLinkingMetadataString() { + return getFilteredLinkingMetadataMap().toString(); } - public static Map getLinkingMetadataAsMap() { - return NewRelic.getAgent().getLinkingMetadata(); + /** + * Gets a map of agent linking metadata after filtering out + * entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered map of agent linking metadata + */ + public static Map getFilteredLinkingMetadataMap() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + Map map = new HashMap<>(); + Set> metadataSet = agentLinkingMetadata.entrySet(); + + for (Map.Entry entry : metadataSet) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { + map.put(key, value); + } + } + return map; + } else { + return Collections.emptyMap(); + } } + /** + * Check if all application_logging features are enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -65,6 +67,11 @@ public static boolean isApplicationLoggingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging metrics feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingMetricsEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -74,6 +81,11 @@ public static boolean isApplicationLoggingMetricsEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging forwarding feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingForwardingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -83,6 +95,11 @@ public static boolean isApplicationLoggingForwardingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging local_decorating feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); // Config value is parsed as a String if it was set by system property or environment variable diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java deleted file mode 100644 index 3b02ccd649..0000000000 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ElementName.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.nr.agent.instrumentation.log4j1; - -public class ElementName { - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String THREAD_NAME = "thread.name"; - public static final String LOG_LEVEL = "log.level"; - public static final String LOGGER_NAME = "logger.name"; - public static final String CLASS_NAME = "class.name"; - public static final String METHOD_NAME = "method.name"; - public static final String LINE_NUMBER = "line.number"; - public static final String THROWABLE = "throwable"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - - private ElementName() {} -} diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java deleted file mode 100644 index bf90224007..0000000000 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/ExceptionUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019. New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.nr.agent.instrumentation.log4j1; - -public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? - - public static String getErrorStack(Throwable throwable) { - if (throwable == null) { - return null; - } - - StackTraceElement[] stack = throwable.getStackTrace(); - return getErrorStack(stack); - } - - public static String getErrorStack(StackTraceElement[] stack) { - return getErrorStack(stack, MAX_STACK_SIZE); - } - - public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { - if (stack == null || stack.length == 0) { - return null; - } - - StringBuilder stackBuilder = new StringBuilder(); - for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { - stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); - } - return stackBuilder.toString(); - } -} diff --git a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java index ceaf8419d6..4d4e2fb434 100644 --- a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java @@ -3,10 +3,9 @@ import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.log4j1.AgentUtil; -import org.apache.log4j.Priority; -import static com.nr.agent.instrumentation.log4j1.AgentUtil.*; +import static com.nr.agent.instrumentation.log4j1.AgentUtil.isApplicationLoggingEnabled; +import static com.nr.agent.instrumentation.log4j1.AgentUtil.isApplicationLoggingMetricsEnabled; @Weave(originalName = "org.apache.log4j.Category") public class Category_Instrumentation { diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index c5ac5d440e..1ae65c4c43 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -1,64 +1,98 @@ package com.nr.agent.instrumentation.log4j2; import com.newrelic.agent.bridge.AgentBridge; -import com.newrelic.api.agent.Config; import com.newrelic.api.agent.NewRelic; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.message.Message; +import java.util.Collections; import java.util.HashMap; import java.util.Map; - -import static com.nr.agent.instrumentation.log4j2.ElementName.CLASS_NAME; -import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_CLASS; -import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_MESSAGE; -import static com.nr.agent.instrumentation.log4j2.ElementName.ERROR_STACK; -import static com.nr.agent.instrumentation.log4j2.ElementName.LOGGER_NAME; -import static com.nr.agent.instrumentation.log4j2.ElementName.LOG_LEVEL; -import static com.nr.agent.instrumentation.log4j2.ElementName.MESSAGE; -import static com.nr.agent.instrumentation.log4j2.ElementName.THROWABLE; -import static com.nr.agent.instrumentation.log4j2.ElementName.TIMESTAMP; +import java.util.Set; public class AgentUtil { - + // Log message attributes + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String LOG_LEVEL = "log.level"; + public static final String UNKNOWN = "UNKNOWN"; + // Linking metadata attributes to filter out + private static final String ENTITY_TYPE = "entity.type"; + private static final String ENTITY_NAME = "entity.name"; + + /** + * Record a LogEvent to be sent to New Relic. + * + * @param event to parse + */ public static void recordNewRelicLogEvent(LogEvent event) { - final String EMPTY_STRING = ""; - if (event != null) { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - HashMap logEventMap = new HashMap<>(agentLinkingMetadata); - Message message = event.getMessage(); - logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); - logEventMap.put(TIMESTAMP, event.getTimeMillis()); - - Level level = event.getLevel(); - logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); - logEventMap.put(LOGGER_NAME, event.getLoggerName()); - logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); - - Throwable throwable = event.getThrown(); - if (throwable != null) { - logEventMap.put(THROWABLE, throwable.toString()); - logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); - logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); - logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); - } + String formattedMessage = message.getFormattedMessage(); + + // Bail out and don't create a LogEvent if log message is empty + if (!formattedMessage.isEmpty()) { + HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); - AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); + logEventMap.put(MESSAGE, formattedMessage); + logEventMap.put(TIMESTAMP, event.getTimeMillis()); + + Level level = event.getLevel(); + String levelName = level.name(); + + if (levelName.isEmpty()) { + logEventMap.put(LOG_LEVEL, UNKNOWN); + } else { + logEventMap.put(LOG_LEVEL, levelName); + } + + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); + } } } - public static String getLinkingMetadataAsString() { - // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string - return NewRelic.getAgent().getLinkingMetadata().toString(); + /** + * Gets a String representing the agent linking metadata after filtering + * out entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered String of agent linking metadata + */ + public static String getFilteredLinkingMetadataString() { + return getFilteredLinkingMetadataMap().toString(); } - public static Map getLinkingMetadataAsMap() { - return NewRelic.getAgent().getLinkingMetadata(); + /** + * Gets a map of agent linking metadata after filtering out + * entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered map of agent linking metadata + */ + public static Map getFilteredLinkingMetadataMap() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + Map map = new HashMap<>(); + Set> metadataSet = agentLinkingMetadata.entrySet(); + + for (Map.Entry entry : metadataSet) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { + map.put(key, value); + } + } + return map; + } else { + return Collections.emptyMap(); + } } + /** + * Check if all application_logging features are enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -68,6 +102,11 @@ public static boolean isApplicationLoggingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging metrics feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingMetricsEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -77,6 +116,11 @@ public static boolean isApplicationLoggingMetricsEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging forwarding feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingForwardingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -86,6 +130,11 @@ public static boolean isApplicationLoggingForwardingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging local_decorating feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); // Config value is parsed as a String if it was set by system property or environment variable diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java deleted file mode 100644 index 8b95637d8e..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ElementName.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.nr.agent.instrumentation.log4j2; - -public class ElementName { - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String THREAD_NAME = "thread.name"; - public static final String LOG_LEVEL = "log.level"; - public static final String LOGGER_NAME = "logger.name"; - public static final String CLASS_NAME = "class.name"; - public static final String METHOD_NAME = "method.name"; - public static final String LINE_NUMBER = "line.number"; - public static final String THROWABLE = "throwable"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - - private ElementName() {} -} diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java deleted file mode 100644 index 0d6643dc3a..0000000000 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/ExceptionUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019. New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.nr.agent.instrumentation.log4j2; - -public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? - - public static String getErrorStack(Throwable throwable) { - if (throwable == null) { - return null; - } - - StackTraceElement[] stack = throwable.getStackTrace(); - return getErrorStack(stack); - } - - public static String getErrorStack(StackTraceElement[] stack) { - return getErrorStack(stack, MAX_STACK_SIZE); - } - - public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { - if (stack == null || stack.length == 0) { - return null; - } - - StringBuilder stackBuilder = new StringBuilder(); - for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { - stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); - } - return stackBuilder.toString(); - } -} diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index a295592268..d6946ce0c6 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -4,10 +4,12 @@ import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.log4j2.AgentUtil; import org.apache.logging.log4j.core.LogEvent; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.*; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingForwardingEnabled; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingMetricsEnabled; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig", type = MatchType.ExactClass) public class LoggerConfig_Instrumentation { diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java index cc5105988f..66b83c04ee 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -3,8 +3,8 @@ import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.log4j2.AgentUtil; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.getFilteredLinkingMetadataString; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; @@ -27,7 +27,7 @@ private void appendAgentMetadata(StringBuilder source) { if (breakLine != -1) { source.replace(breakLine, breakLine + 1, ""); } - source.append(" NR-LINKING-METADATA: ").append(AgentUtil.getLinkingMetadataAsString()).append("\n"); + source.append(" NR-LINKING-METADATA: ").append(getFilteredLinkingMetadataString()).append("\n"); } } \ No newline at end of file diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 5c200475ac..f579ecd7bc 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -2,48 +2,62 @@ import com.newrelic.api.agent.NewRelic; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import java.util.Set; public class AgentUtil { + // Log message attributes + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String LOG_LEVEL = "log.level"; + public static final String UNKNOWN = "UNKNOWN"; + // Linking metadata attributes to filter out + private static final String ENTITY_TYPE = "entity.type"; + private static final String ENTITY_NAME = "entity.name"; - // TODO figure out how to recordNewRelicLogEvent for JUL -// public static void recordNewRelicLogEvent(LogEvent event) { -// final String EMPTY_STRING = ""; -// -// if (event != null) { -// Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); -// HashMap logEventMap = new HashMap<>(agentLinkingMetadata); -// -// Message message = event.getMessage(); -// logEventMap.put(MESSAGE, message != null ? message.getFormattedMessage() : EMPTY_STRING); -// logEventMap.put(TIMESTAMP, event.getTimeMillis()); -// -// Level level = event.getLevel(); -// logEventMap.put(LOG_LEVEL, level != null ? level.name() : EMPTY_STRING); -// logEventMap.put(LOGGER_NAME, event.getLoggerName()); -// logEventMap.put(CLASS_NAME, event.getLoggerFqcn()); -// -// Throwable throwable = event.getThrown(); -// if (throwable != null) { -// logEventMap.put(THROWABLE, throwable.toString()); -// logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); -// logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); -// logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); -// } -// -// AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); -// } -// } - - public static String getLinkingMetadataAsString() { - // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string - return NewRelic.getAgent().getLinkingMetadata().toString(); + /** + * Gets a String representing the agent linking metadata after filtering + * out entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered String of agent linking metadata + */ + public static String getFilteredLinkingMetadataString() { + return getFilteredLinkingMetadataMap().toString(); } - public static Map getLinkingMetadataAsMap() { - return NewRelic.getAgent().getLinkingMetadata(); + /** + * Gets a map of agent linking metadata after filtering out + * entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered map of agent linking metadata + */ + public static Map getFilteredLinkingMetadataMap() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + Map map = new HashMap<>(); + Set> metadataSet = agentLinkingMetadata.entrySet(); + + for (Map.Entry entry : metadataSet) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { + map.put(key, value); + } + } + return map; + } else { + return Collections.emptyMap(); + } } + /** + * Check if all application_logging features are enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -53,6 +67,11 @@ public static boolean isApplicationLoggingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging metrics feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingMetricsEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -62,6 +81,11 @@ public static boolean isApplicationLoggingMetricsEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging forwarding feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingForwardingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -71,6 +95,11 @@ public static boolean isApplicationLoggingForwardingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging local_decorating feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); // Config value is parsed as a String if it was set by system property or environment variable diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java deleted file mode 100644 index fd16eb473a..0000000000 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ElementName.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.nr.instrumentation.jul; - -public class ElementName { - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String THREAD_NAME = "thread.name"; - public static final String LOG_LEVEL = "log.level"; - public static final String LOGGER_NAME = "logger.name"; - public static final String CLASS_NAME = "class.name"; - public static final String METHOD_NAME = "method.name"; - public static final String LINE_NUMBER = "line.number"; - public static final String THROWABLE = "throwable"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - - private ElementName() {} -} diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java deleted file mode 100644 index 43e3705b9a..0000000000 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/ExceptionUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019. New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.nr.instrumentation.jul; - -public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? - - public static String getErrorStack(Throwable throwable) { - if (throwable == null) { - return null; - } - - StackTraceElement[] stack = throwable.getStackTrace(); - return getErrorStack(stack); - } - - public static String getErrorStack(StackTraceElement[] stack) { - return getErrorStack(stack, MAX_STACK_SIZE); - } - - public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { - if (stack == null || stack.length == 0) { - return null; - } - - StringBuilder stackBuilder = new StringBuilder(); - for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { - stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); - } - return stackBuilder.toString(); - } -} diff --git a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java index 47c0573067..0c6d1a87b1 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java +++ b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java @@ -3,13 +3,9 @@ import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.instrumentation.jul.AgentUtil; -import java.util.logging.Filter; -import java.util.logging.Level; -import java.util.logging.LogRecord; - -import static com.nr.instrumentation.jul.AgentUtil.*; +import static com.nr.instrumentation.jul.AgentUtil.isApplicationLoggingEnabled; +import static com.nr.instrumentation.jul.AgentUtil.isApplicationLoggingMetricsEnabled; @Weave(originalName = "java.util.logging.Logger") public class Logger_Instrumentation { diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 58aa882bfd..9d03ca4839 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -6,9 +6,11 @@ import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.*; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getFilteredLinkingMetadataString; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingEnabled; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingForwardingEnabled; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) @@ -35,7 +37,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str if (isApplicationLoggingLocalDecoratingEnabled()) { // Append New Relic linking metadata from agent to log message - this.message = message + " NR-LINKING-METADATA: " + getLinkingMetadataAsString(); + this.message = message + " NR-LINKING-METADATA: " + getFilteredLinkingMetadataString(); } else { this.message = message; } @@ -58,7 +60,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str if (isApplicationLoggingForwardingEnabled()) { // Record and send LogEvent to New Relic - recordNewRelicLogEvent(message, timeStamp, level, logger, fqcn, throwable); + recordNewRelicLogEvent(message, timeStamp, level); } } else { Weaver.callOriginal(); diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index e033641f6d..8ad3018397 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -1,54 +1,89 @@ package com.nr.agent.instrumentation.logbackclassic12; import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; +import java.util.Collections; import java.util.HashMap; import java.util.Map; - -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.CLASS_NAME; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_CLASS; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_MESSAGE; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.ERROR_STACK; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.LOGGER_NAME; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.LOG_LEVEL; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.MESSAGE; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.THROWABLE; -import static com.nr.agent.instrumentation.logbackclassic12.ElementName.TIMESTAMP; +import java.util.Set; public class AgentUtil { + // Log message attributes + public static final String MESSAGE = "message"; + public static final String TIMESTAMP = "timestamp"; + public static final String LOG_LEVEL = "log.level"; + public static final String UNKNOWN = "UNKNOWN"; + // Linking metadata attributes to filter out + private static final String ENTITY_TYPE = "entity.type"; + private static final String ENTITY_NAME = "entity.name"; - public static void recordNewRelicLogEvent(String message, long timeStampMillis, Level level, Logger logger, String fqcn, Throwable throwable) { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - HashMap logEventMap = new HashMap<>(agentLinkingMetadata); + /** + * Record a LogEvent to be sent to New Relic. + * + * @param message log message + * @param timeStampMillis log timestamp + * @param level log level + */ + public static void recordNewRelicLogEvent(String message, long timeStampMillis, Level level) { + // Bail out and don't create a LogEvent if log message is empty + if (!message.isEmpty()) { + HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); + logEventMap.put(MESSAGE, message); + logEventMap.put(TIMESTAMP, timeStampMillis); - logEventMap.put(MESSAGE, message); - logEventMap.put(TIMESTAMP, timeStampMillis); - logEventMap.put(LOG_LEVEL, level); - logEventMap.put(LOGGER_NAME, logger.getName()); - logEventMap.put(CLASS_NAME, fqcn); + if (level.toString().isEmpty()) { + logEventMap.put(LOG_LEVEL, UNKNOWN); + } else { + logEventMap.put(LOG_LEVEL, level); + } - if (throwable != null) { - logEventMap.put(THROWABLE, throwable.toString()); - logEventMap.put(ERROR_CLASS, throwable.getClass().getName()); - logEventMap.put(ERROR_MESSAGE, throwable.getMessage()); - logEventMap.put(ERROR_STACK, ExceptionUtil.getErrorStack(throwable)); + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } - - AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } - public static Map getLinkingMetadataAsMap() { - return NewRelic.getAgent().getLinkingMetadata(); + /** + * Gets a String representing the agent linking metadata after filtering + * out entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered String of agent linking metadata + */ + public static String getFilteredLinkingMetadataString() { + return getFilteredLinkingMetadataMap().toString(); } - public static String getLinkingMetadataAsString() { - // TODO might need to filter the map entries to remove some (e.g. entity.*) and/or create a differently formatted string - return NewRelic.getAgent().getLinkingMetadata().toString(); + /** + * Gets a map of agent linking metadata after filtering out + * entity.type, entity.name, and any attributes with an empty value. + * + * @return Filtered map of agent linking metadata + */ + public static Map getFilteredLinkingMetadataMap() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + Map map = new HashMap<>(); + Set> metadataSet = agentLinkingMetadata.entrySet(); + + for (Map.Entry entry : metadataSet) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { + map.put(key, value); + } + } + return map; + } else { + return Collections.emptyMap(); + } } + /** + * Check if all application_logging features are enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -58,6 +93,11 @@ public static boolean isApplicationLoggingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging metrics feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingMetricsEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -67,6 +107,11 @@ public static boolean isApplicationLoggingMetricsEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging forwarding feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingForwardingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); // Config value is parsed as a String if it was set by system property or environment variable @@ -76,6 +121,11 @@ public static boolean isApplicationLoggingForwardingEnabled() { return (Boolean) configValue; } + /** + * Check if the application_logging local_decorating feature is enabled. + * + * @return true if enabled, else false + */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); // Config value is parsed as a String if it was set by system property or environment variable diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java deleted file mode 100644 index ef6e313c47..0000000000 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ElementName.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.nr.agent.instrumentation.logbackclassic12; - -public class ElementName { - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String THREAD_NAME = "thread.name"; - public static final String LOG_LEVEL = "log.level"; - public static final String LOGGER_NAME = "logger.name"; - public static final String CLASS_NAME = "class.name"; - public static final String METHOD_NAME = "method.name"; - public static final String LINE_NUMBER = "line.number"; - public static final String THROWABLE = "throwable"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - - private ElementName() {} -} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java deleted file mode 100644 index 106976d8b0..0000000000 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/ExceptionUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019. New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.nr.agent.instrumentation.logbackclassic12; - -public class ExceptionUtil { - public static final int MAX_STACK_SIZE = 1000; // TODO figure out size limit. limited by ConfigConstant.MAX_LOG_EVENT_ATTRIBUTE_SIZE? - - public static String getErrorStack(Throwable throwable) { - if (throwable == null) { - return null; - } - - StackTraceElement[] stack = throwable.getStackTrace(); - return getErrorStack(stack); - } - - public static String getErrorStack(StackTraceElement[] stack) { - return getErrorStack(stack, MAX_STACK_SIZE); - } - - public static String getErrorStack(StackTraceElement[] stack, Integer maxStackSize) { - if (stack == null || stack.length == 0) { - return null; - } - - StringBuilder stackBuilder = new StringBuilder(); - for (int i = 0; i < Math.min(maxStackSize, stack.length); i++) { - stackBuilder.append(" at ").append(stack[i].toString()).append("\n"); - } - return stackBuilder.toString(); - } -} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java index 2af3870eef..425ff5a655 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ConfigConstant.java @@ -10,5 +10,5 @@ public class ConfigConstant { public static final int MAX_USER_ATTRIBUTES = 64; public static final int MAX_USER_ATTRIBUTE_SIZE = 255; // Size in bytes - public static final int MAX_LOG_EVENT_ATTRIBUTE_SIZE = 32000; // Size in bytes + public static final int MAX_LOG_EVENT_ATTRIBUTE_SIZE = 32767; // Size in bytes } From d922bab123a22d94f3229752d1273bb5e8b63ffc Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Feb 2022 20:33:58 -0800 Subject: [PATCH 36/96] Fix WeaveViolation --- .../spi/LoggingEvent_Instrumentation.java | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 9d03ca4839..3caf45eb22 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -28,42 +28,39 @@ public class LoggingEvent_Instrumentation { public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, String message, Throwable throwable, Object[] argArray) { // Do nothing if application_logging.enabled: false - if (isApplicationLoggingEnabled()) { - this.fqnOfLoggerClass = fqcn; - this.loggerName = logger.getName(); - this.loggerContext = logger.getLoggerContext(); - this.loggerContextVO = loggerContext.getLoggerContextRemoteView(); - this.level = level; + this.fqnOfLoggerClass = fqcn; + this.loggerName = logger.getName(); + this.loggerContext = logger.getLoggerContext(); + this.loggerContextVO = loggerContext.getLoggerContextRemoteView(); + this.level = level; - if (isApplicationLoggingLocalDecoratingEnabled()) { - // Append New Relic linking metadata from agent to log message - this.message = message + " NR-LINKING-METADATA: " + getFilteredLinkingMetadataString(); - } else { - this.message = message; - } + boolean applicationLoggingEnabled = isApplicationLoggingEnabled(); + if (applicationLoggingEnabled && isApplicationLoggingLocalDecoratingEnabled()) { + // Append New Relic linking metadata from agent to log message + this.message = message + " NR-LINKING-METADATA: " + getFilteredLinkingMetadataString(); + } else { + this.message = message; + } - this.argumentArray = argArray; + this.argumentArray = argArray; - if (throwable == null) { - throwable = extractThrowableAnRearrangeArguments(argArray); - } + if (throwable == null) { + throwable = extractThrowableAnRearrangeArguments(argArray); + } - if (throwable != null) { - this.throwableProxy = new ThrowableProxy(throwable); - LoggerContext lc = logger.getLoggerContext(); - if (lc.isPackagingDataEnabled()) { - this.throwableProxy.calculatePackagingData(); - } + if (throwable != null) { + this.throwableProxy = new ThrowableProxy(throwable); + LoggerContext lc = logger.getLoggerContext(); + if (lc.isPackagingDataEnabled()) { + this.throwableProxy.calculatePackagingData(); } + } - timeStamp = System.currentTimeMillis(); + timeStamp = System.currentTimeMillis(); - if (isApplicationLoggingForwardingEnabled()) { - // Record and send LogEvent to New Relic - recordNewRelicLogEvent(message, timeStamp, level); - } - } else { - Weaver.callOriginal(); + if (applicationLoggingEnabled && isApplicationLoggingForwardingEnabled()) { + // Record and send LogEvent to New Relic + recordNewRelicLogEvent(message, timeStamp, level); } } From 3d7c764e556ac42705e9570772226aca6716c752 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 18 Feb 2022 10:21:38 -0800 Subject: [PATCH 37/96] Add instrumentation enabled supportability metrics --- .../core/config/LoggerConfig_Instrumentation.java | 14 ++++++++++++++ .../logback/classic/Logger_Instrumentation.java | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index d6946ce0c6..aeca4ff506 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -2,10 +2,14 @@ import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.NewField; import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.WeaveAllConstructors; import com.newrelic.api.agent.weaver.Weaver; import org.apache.logging.log4j.core.LogEvent; +import java.util.concurrent.atomic.AtomicBoolean; + import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingForwardingEnabled; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingMetricsEnabled; @@ -13,6 +17,16 @@ @Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig", type = MatchType.ExactClass) public class LoggerConfig_Instrumentation { + @NewField + public static AtomicBoolean instrumented = new AtomicBoolean(false); + + @WeaveAllConstructors + public LoggerConfig_Instrumentation() { + // Generate the instrumentation module supportability metric only once + if (!instrumented.getAndSet(true)) { + NewRelic.incrementCounter("Supportability/Logging/enabled/Java/Log4j2"); + } + } protected void callAppenders(LogEvent event) { // Do nothing if application_logging.enabled: false diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index 2356036136..4f0a944997 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -2,13 +2,27 @@ import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; +import com.newrelic.api.agent.weaver.NewField; import com.newrelic.api.agent.weaver.Weave; +import com.newrelic.api.agent.weaver.WeaveAllConstructors; import com.newrelic.api.agent.weaver.Weaver; import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; import org.slf4j.Marker; +import java.util.concurrent.atomic.AtomicBoolean; + @Weave(originalName = "ch.qos.logback.classic.Logger", type = MatchType.ExactClass) public abstract class Logger_Instrumentation { + @NewField + public static AtomicBoolean instrumented = new AtomicBoolean(false); + + @WeaveAllConstructors + Logger_Instrumentation() { + // Generate the instrumentation module supportability metric only once + if (!instrumented.getAndSet(true)) { + NewRelic.incrementCounter("Supportability/Logging/enabled/Java/LogbackClassic1.2"); + } + } private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { From 19dca5a38227f30201fe2ec46b88a22c39d5c01b Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 18 Feb 2022 15:39:16 -0800 Subject: [PATCH 38/96] Some cleanup --- .../java/com/newrelic/agent/HarvestServiceImpl.java | 9 +++++---- .../agent/config/ApplicationLoggingConfigImpl.java | 10 ++-------- .../config/ApplicationLoggingForwardingConfig.java | 2 +- .../ApplicationLoggingLocalDecoratingConfig.java | 2 +- .../agent/config/ApplicationLoggingMetricsConfig.java | 2 +- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index af06f0e8f6..6ff67fbeb8 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -98,14 +98,15 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { boolean isSpanEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(SPAN_EVENT_DATA); boolean isLogEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(LOG_EVENT_DATA); - // The event_harvest_config received from server-side during the connect lifecycle contains config for error_event_data, analytic_event_data, and custom_event_data + // The event_harvest_config.harvest_limits received from server-side during the connect lifecycle contains config for error_event_data, + // analytic_event_data, and custom_event_data TODO The event_harvest_config.harvest_limits should also include log_event_data endpoint at some point... if (eventHarvestConfig != null && !isSpanEventEndpoint) { Agent.LOG.log(Level.FINE, "event_harvest_config from collector is: {0} samples stored for {1}", maxSamplesStored, tracker.harvestable.getEndpointMethodName()); Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); - // TODO set harvest_limits for log_event_data endpoint + Long harvestLimit; - // TODO THIS IS A HACK! Real limit for log_event_data endpoint should come from server side!!! + // FIXME THIS IS A HACK! Real harvestLimit for log_event_data endpoint should come from server side event_harvest_config.harvest_limits!!! if (isLogEventEndpoint) { int maxSamplesStoredLogEvents = ServiceFactory.getLogSenderService().getMaxSamplesStored(); harvestLimit = (long) maxSamplesStoredLogEvents; @@ -114,7 +115,7 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { } if (harvestLimit != null) { maxSamplesStored = harvestLimit.intValue(); - reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); + reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period ServiceFactory.getStatsService().doStatsWork( StatsWorks.getRecordMetricWork(MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS, reportPeriodInMillis / 1000)); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java index 088b20bc67..77cd1f9ab6 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ @@ -30,7 +30,7 @@ public class ApplicationLoggingConfigImpl extends BaseConfig implements Applicat public static final String FORWARDING = "forwarding"; public static final String LOCAL_DECORATING = "local_decorating"; - public static final boolean DEFAULT_ENABLED = false; // TODO what is default? + public static final boolean DEFAULT_ENABLED = false; public static final String ENABLED = "enabled"; private final ApplicationLoggingMetricsConfig applicationLoggingMetricsConfig; @@ -62,12 +62,6 @@ private ApplicationLoggingForwardingConfig createApplicationLoggingForwardingCon return new ApplicationLoggingForwardingConfig(forwardingProps, SYSTEM_PROPERTY_ROOT, highSecurity); } -// public boolean initForwardingEnabled(boolean highSecurity) { -// boolean storedMoreThan0 = maxSamplesStored > 0; -// Boolean configEnabled = getProperty(FORWARDING_ENABLED_PROP, DEFAULT_FORWARDING_ENABLED); -// return isEnabled && !highSecurity && storedMoreThan0 && configEnabled; -// } - static ApplicationLoggingConfigImpl createApplicationLoggingConfig(Map settings, boolean highSecurity) { if (settings == null) { settings = Collections.emptyMap(); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java index 75b1b0b9d1..edd1d42cfc 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java index b27e98df46..f0289b6534 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfig.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java index 1433ba1b57..39a4aebdf3 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ From 06ee3c1d6b3a7a203c7d2de7d4350282e1d2a9cc Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 18 Feb 2022 15:49:02 -0800 Subject: [PATCH 39/96] Some more cleanup --- .../agent/service/logging/LogSenderServiceImpl.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 159302a887..e9aaa07138 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -73,14 +73,14 @@ public void dispatcherTransactionStarted(Transaction transaction) { @Override public void dispatcherTransactionFinished(TransactionData transactionData, TransactionStats transactionStats) { - // FIXME not sure this is a great idea to store log events for the duration of a transaction... + // Get log events from the transaction when it is finished TransactionLogs data = (TransactionLogs) transactionData.getLogEventData(); storeEvents(transactionData.getApplicationName(), transactionData.getPriority(), data.events); } @Override public void dispatcherTransactionCancelled(Transaction transaction) { - // FIXME not sure this is a great idea to store log events for the duration of a transaction... + // Get log events from the transaction when it is canceled // Even if the transaction is canceled we still want to send up any events that were held in it TransactionLogs data = (TransactionLogs) transaction.getLogEventData(); storeEvents(transaction.getApplicationName(), transaction.getPriority(), data.events); @@ -151,7 +151,7 @@ public boolean isEnabled() { */ @Override protected void doStart() throws Exception { - // TODO it's not clear that log sender events should be tied to transactions in any way + // Register transaction listener to associate log events with transaction lifecycle ServiceFactory.getTransactionService().addTransactionListener(transactionListener); ServiceFactory.getConfigService().addIAgentConfigListener(configListener); } @@ -163,7 +163,6 @@ protected void doStart() throws Exception { @Override protected void doStop() throws Exception { removeHarvestables(); - // TODO it's not clear that log sender events should be tied to transactions in any way ServiceFactory.getTransactionService().removeTransactionListener(transactionListener); ServiceFactory.getConfigService().removeIAgentConfigListener(configListener); reservoirForApp.clear(); @@ -190,8 +189,6 @@ public void recordLogEvent(Map attributes) { } Transaction transaction = ServiceFactory.getTransactionService().getTransaction(false); - // FIXME perhaps ignore transaction status and just always send log events... - // what is the benefit of storing them on the transaction? Sampling? // Not in a Transaction or an existing Transaction is not in progress or is ignored if (transaction == null || !transaction.isInProgress() || transaction.isIgnore()) { String applicationName = ServiceFactory.getRPMService().getApplicationName(); @@ -209,7 +206,7 @@ public void recordLogEvent(Map attributes) { createAndStoreEvent(applicationName, attributes); // In a Transaction that is in progress and not ignored } else { - // FIXME not sure this is a great idea to store log events for the duration of a transaction... + // Store log events on the transaction transaction.getLogEventData().recordLogEvent(attributes); } MetricNames.recordApiSupportabilityMetric(MetricNames.SUPPORTABILITY_API_RECORD_LOG_EVENT); From c7a1be67a825137202070c16b1372b626a752ca4 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 18 Feb 2022 16:32:12 -0800 Subject: [PATCH 40/96] Refactoring --- .../src/main/java/com/newrelic/agent/model/LogEvent.java | 5 ++--- .../introspec/internal/IntrospectorLogSenderService.java | 2 +- .../agent/service/logging/LogSenderServiceImpl.java | 4 +--- .../java/com/newrelic/agent/transport/DataSenderImpl.java | 6 ++---- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java index bb1f9604f1..41fc3243a6 100644 --- a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -20,9 +20,8 @@ public class LogEvent extends AnalyticsEvent implements JSONStreamAware { private volatile float mutablePriority; - // FIXME probably don't need to pass timestamp as we use the value from the log event captured in the library instrumentation - public LogEvent(long timestamp, Map attributes, float priority) { - super(LOG_EVENT_TYPE, timestamp, priority, attributes); + public LogEvent(Map attributes, float priority) { + super(LOG_EVENT_TYPE, System.currentTimeMillis(), priority, attributes); this.mutablePriority = priority; } diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java index bdc25ef113..f643d5471e 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -82,7 +82,7 @@ public boolean isStoppedOrStopping() { public void recordLogEvent(Map attributes) { if (AnalyticsEvent.isValidType(LOG_EVENT_TYPE)) { Map atts = Maps.newHashMap(attributes); - LogEvent event = new LogEvent(System.currentTimeMillis(), atts, DistributedTraceServiceImpl.nextTruncatedFloat()); + LogEvent event = new LogEvent(atts, DistributedTraceServiceImpl.nextTruncatedFloat()); storeEvent("TestApp", event); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index e9aaa07138..25597a6bae 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -472,9 +472,7 @@ private static String mapInternString(String value) { */ private static LogEvent createValidatedEvent(Map attributes) { Map userAttributes = new HashMap<>(attributes.size()); - // FIXME LogEvent constructor only needs the timestamp for the AnalyticsEvent super class but it won't - // actually be added to the LogEvent as it isn't needed. We use the timestamp captured from the log library. - LogEvent event = new LogEvent(System.currentTimeMillis(), userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); + LogEvent event = new LogEvent(userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); // Now add the attributes from the argument map to the event using an AttributeSender. // An AttributeSender is the way to reuse all the existing attribute validations. We diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 8108d968ac..d444078a81 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -338,7 +338,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect @Override public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { - sendLogEventsForReservoir(CollectorMethods.LOG_EVENT_DATA, compressedEncoding, reservoirSize, eventsSeen, events); + sendLogEventsForReservoir(CollectorMethods.LOG_EVENT_DATA, compressedEncoding, events); } @Override @@ -366,9 +366,7 @@ private void sendAnalyticEventsForR // Sends LogEvent data in the MELT format for logs // https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#log-attribute-example - private void sendLogEventsForReservoir(String method, String encoding, int reservoirSize, int eventsSeen, - Collection events) throws Exception { - // FIXME remove unused parameters + private void sendLogEventsForReservoir(String method, String encoding, Collection events) throws Exception { Object runId = agentRunId; if (runId == NO_AGENT_RUN_ID || events.isEmpty()) { return; From a94890fee50dfad05edb32be5776e509d275fb0b Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 22 Feb 2022 11:50:51 -0800 Subject: [PATCH 41/96] Update copyright headers. Remove some comments. --- .../src/main/java/com/newrelic/agent/bridge/NoOpLogs.java | 2 +- .../src/main/java/com/newrelic/api/agent/Logs.java | 2 +- .../src/main/java/com/newrelic/agent/model/LogEvent.java | 2 +- .../introspec/internal/IntrospectorLogSenderService.java | 2 +- .../com/nr/agent/instrumentation/log4j1/AgentUtil.java | 7 +++++++ .../java/org/apache/log4j/Category_Instrumentation.java | 7 +++++++ .../com/nr/agent/instrumentation/log4j2/AgentUtil.java | 7 +++++++ .../log4j/core/config/LoggerConfig_Instrumentation.java | 7 +++++++ .../core/layout/StringBuilderEncoder_Instrumentation.java | 7 +++++++ .../main/java/com/nr/instrumentation/jul/AgentUtil.java | 7 +++++++ .../java/java/util/logging/Logger_Instrumentation.java | 7 +++++++ .../ch/qos/logback/classic/Logger_Instrumentation.java | 7 +++++++ .../logback/classic/spi/LoggingEvent_Instrumentation.java | 7 +++++++ .../agent/instrumentation/logbackclassic12/AgentUtil.java | 7 +++++++ .../src/main/java/com/newrelic/agent/RPMService.java | 1 - .../newrelic/agent/config/ApplicationLoggingConfig.java | 2 +- .../agent/service/logging/LogSenderHarvestableImpl.java | 2 +- .../newrelic/agent/service/logging/LogSenderService.java | 2 +- .../agent/service/logging/LogSenderServiceImpl.java | 2 +- .../java/com/newrelic/agent/transport/DataSenderImpl.java | 2 +- 20 files changed, 79 insertions(+), 10 deletions(-) diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java index 9feeadb058..7825027abc 100644 --- a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java index f0b262fb4f..bd8170e9c5 100644 --- a/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java +++ b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java index 41fc3243a6..ded1bfce80 100644 --- a/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java +++ b/agent-model/src/main/java/com/newrelic/agent/model/LogEvent.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java index f643d5471e..e839f439d9 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index bb98da3ff2..f429b3d9bf 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package com.nr.agent.instrumentation.log4j1; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java index 4d4e2fb434..ddc1e62802 100644 --- a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package org.apache.log4j; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 1ae65c4c43..1c437c3811 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package com.nr.agent.instrumentation.log4j2; import com.newrelic.agent.bridge.AgentBridge; diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index aeca4ff506..8c7171d7da 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package org.apache.logging.log4j.core.config; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java index 66b83c04ee..122c5f5c03 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package org.apache.logging.log4j.core.layout; import com.newrelic.api.agent.weaver.MatchType; diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index f579ecd7bc..66b4c3a68a 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package com.nr.instrumentation.jul; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java index 0c6d1a87b1..bef11fa70f 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java +++ b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package java.util.logging; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index 4f0a944997..e55878f657 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package ch.qos.logback.classic; import com.newrelic.api.agent.NewRelic; diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 3caf45eb22..e4253753a5 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package ch.qos.logback.classic.spi; import ch.qos.logback.classic.Level; diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 8ad3018397..4b7aa5f3de 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -1,3 +1,10 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + package com.nr.agent.instrumentation.logbackclassic12; import ch.qos.logback.classic.Level; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java index fc09b5832d..3a2b5ef3d9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java @@ -411,7 +411,6 @@ public void sendErrorData(List errors) { // In case of 413 status code, cut the size of the payload in half and try again if (e.isRequestPayloadTooLarge()) { // This will halve the errors payload. If the payload only has 1 item left it will be cut to 0 - // TODO should we follow this halving behavior for sendLogEvents? sendErrorData(new ArrayList<>(errors.subList(0, errors.size() / 2))); } else { throw e; // Otherwise re-throw the error so it can be logged diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java index b72bf1a7e9..924a3f59f2 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java index 68d2a89d53..659c0ac0aa 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderHarvestableImpl.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java index 34e9e32754..c31ceb8a14 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 25597a6bae..3d661691b1 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -1,6 +1,6 @@ /* * - * * Copyright 2020 New Relic Corporation. All rights reserved. + * * Copyright 2022 New Relic Corporation. All rights reserved. * * SPDX-License-Identifier: Apache-2.0 * */ diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index d444078a81..3e5ea0f917 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -572,7 +572,7 @@ private ReadResult connectAndSend(String host, String method, String encoding, S /* * We don't enforce max_payload_size_in_bytes for error_data (aka error traces). Instead we halve the - * payload and try again. See RPMService sendErrorData // TODO does this apply to log data as well? + * payload and try again. See RPMService sendErrorData */ if (data.length > maxPayloadSizeInBytes && !method.equals(CollectorMethods.ERROR_DATA)) { ServiceFactory.getStatsService().doStatsWork(StatsWorks.getIncrementCounterWork( From e682e570d5c79e6d04519b781d386698f3d13902 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 22 Feb 2022 16:43:41 -0800 Subject: [PATCH 42/96] Update default yaml --- .../src/main/resources/newrelic.yml | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 63f41a5cb0..3bd277a36c 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -83,17 +83,41 @@ common: &default_settings # Default is the logs directory in the newrelic.jar parent directory. #log_file_path: - # Provides the ability to have the agent forward application logs to New Relic - # in a format that is enhanced with agent metadata. This allows the logs to be - # linked with specific transactions and errors. + # Provides the ability to forward application logs to New Relic, generate log usage metrics, + # and decorate local application log files with agent metadata for use with third party log forwarders. + # The application_logging.forwarding and application_logging.local_decorating should not be used together. application_logging: + + # Application logging features are disabled by default. Set this to true to enable + # control over the individual forwarding, local log decorating, and metrics features. + # Default is false. enabled: false + + # The agent will automatically forward application logs to New Relic in + # a format that includes agent metadata for linking them to traces and errors. forwarding: + + # When true, application logs will be forwarded to New Relic. The default is false. enabled: false + + # Application log events are collected up to the configured amount. Afterwards, + # events are sampled to maintain an even distribution across the harvest cycle. + # Default is 2000. Setting to 0 will disable. max_samples_stored: 2000 - local_decorating: - enabled: false + + # The agent will generate metrics to indicate the number of + # application log events occurring at each distinct log level. metrics: + + # When true, application log metrics will be reported. The default is false. + enabled: false + + # The agent will add linking metadata to each log line in your application log files. + # This feature should only be used if you want to use a third party log forwarder, instead + # of the agent's built-in forwarding feature, to send your application log events to New Relic. + local_decorating: + + # When true, the agent will decorate your application log files with linking metadata. The default is false. enabled: false # Proxy settings for connecting to the New Relic server: From e6b6338e28d40dd9164f6e9368d6e1d4eb831db7 Mon Sep 17 00:00:00 2001 From: Jason Keller Date: Tue, 22 Feb 2022 16:48:17 -0800 Subject: [PATCH 43/96] Remove TODO comment --- .../src/main/java/com/newrelic/SpanEventSender.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java b/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java index e2113bbb95..05e624995b 100644 --- a/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java +++ b/infinite-tracing/src/main/java/com/newrelic/SpanEventSender.java @@ -99,8 +99,7 @@ void writeToObserver(ClientCallStreamObserver observer, V1.Span span) { logger.log(Level.SEVERE, t, "Unable to send span."); throw t; } - // TODO record data usage metrics here for the INFINITE_TRACING destination aggregator.incrementCounter("Supportability/InfiniteTracing/Span/Sent"); } -} \ No newline at end of file +} From bfc5c4690c39a824d3d81667e59b516089e4e789 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 22 Feb 2022 17:08:28 -0800 Subject: [PATCH 44/96] Remove unused variable --- .../main/java/com/newrelic/agent/transport/DataSenderImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 3e5ea0f917..0775d21b50 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -94,7 +94,6 @@ public class DataSenderImpl implements DataSender { // Destinations for agent data private static final String COLLECTOR = "Collector"; - private static final String OTLP = "OTLP"; // Not currently supported by Java agent // As of P17 these are the only agent endpoints that actually contain data in the response payload for a successful request private static final Set METHODS_WITH_RESPONSE_BODY = ImmutableSet.of( From 88425f5003f5f951ed2fcf760111ba788fe9afab Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Tue, 22 Feb 2022 18:16:25 -0800 Subject: [PATCH 45/96] Fix FakeExtensionAgent --- .../com/newrelic/agent/extension/FakeExtensionAgent.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/functional_test/src/test/java/com/newrelic/agent/extension/FakeExtensionAgent.java b/functional_test/src/test/java/com/newrelic/agent/extension/FakeExtensionAgent.java index 35a5f7b040..8e26b041e3 100644 --- a/functional_test/src/test/java/com/newrelic/agent/extension/FakeExtensionAgent.java +++ b/functional_test/src/test/java/com/newrelic/agent/extension/FakeExtensionAgent.java @@ -13,6 +13,7 @@ import com.newrelic.api.agent.Config; import com.newrelic.api.agent.Insights; import com.newrelic.api.agent.Logger; +import com.newrelic.api.agent.Logs; import com.newrelic.api.agent.MetricAggregator; import com.newrelic.api.agent.TraceMetadata; @@ -59,4 +60,9 @@ public Logger getLogger() { @Override public boolean ignoreIfUnstartedAsyncContext(Object activityContext) { throw new RuntimeException(); } + + @Override + public Logs getLogSender() { + throw new RuntimeException(); + } } From e9e5afcf88552c82e193f6b2c56354739e46c446 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Feb 2022 13:37:38 -0800 Subject: [PATCH 46/96] Fix NPE --- .../nr/agent/instrumentation/log4j1/AgentUtil.java | 13 +++++++++---- .../nr/agent/instrumentation/log4j2/AgentUtil.java | 13 +++++++++---- .../java/com/nr/instrumentation/jul/AgentUtil.java | 13 +++++++++---- .../instrumentation/logbackclassic12/AgentUtil.java | 13 +++++++++---- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index f429b3d9bf..2151cca046 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -23,6 +23,11 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Enabled defaults + private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** * Gets a String representing the agent linking metadata after filtering @@ -66,7 +71,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -80,7 +85,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -94,7 +99,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -108,7 +113,7 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 1c437c3811..ffded56e1f 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -27,6 +27,11 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Enabled defaults + private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** * Record a LogEvent to be sent to New Relic. @@ -101,7 +106,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -115,7 +120,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -129,7 +134,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -143,7 +148,7 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 66b4c3a68a..691ccf5e4f 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -23,6 +23,11 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Enabled defaults + private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** * Gets a String representing the agent linking metadata after filtering @@ -66,7 +71,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -80,7 +85,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -94,7 +99,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -108,7 +113,7 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 4b7aa5f3de..d45dec8261 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -25,6 +25,11 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Enabled defaults + private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** * Record a LogEvent to be sent to New Relic. @@ -92,7 +97,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -106,7 +111,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -120,7 +125,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); @@ -134,7 +139,7 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled"); + Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); // Config value is parsed as a String if it was set by system property or environment variable if (configValue instanceof String) { return Boolean.parseBoolean((String) configValue); From d5275f96719634a3534ea62c3a72400d62b29952 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Feb 2022 16:04:53 -0800 Subject: [PATCH 47/96] Fix JavaUtilLoggerTest --- .../{LoggerTest.java => JavaUtilLoggerTest.java} | 11 +++++++---- .../resources/com/newrelic/agent/config/newrelic.yml | 10 ++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) rename functional_test/src/test/java/test/newrelic/test/agent/{LoggerTest.java => JavaUtilLoggerTest.java} (94%) diff --git a/functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java b/functional_test/src/test/java/test/newrelic/test/agent/JavaUtilLoggerTest.java similarity index 94% rename from functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java rename to functional_test/src/test/java/test/newrelic/test/agent/JavaUtilLoggerTest.java index 0a754f5e92..0d6c6821b8 100644 --- a/functional_test/src/test/java/test/newrelic/test/agent/LoggerTest.java +++ b/functional_test/src/test/java/test/newrelic/test/agent/JavaUtilLoggerTest.java @@ -14,7 +14,10 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; -public class LoggerTest { +/** + * Test for com.newrelic.instrumentation.java.logging-jdk8 instrumentation + */ +public class JavaUtilLoggerTest { private static final String CAPTURED = "This log message should be captured"; private static final String NOT_CAPTURED = "This message should NOT be captured"; @@ -28,7 +31,7 @@ public void setup() { @Test public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { // Given - final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + final Logger logger = Logger.getLogger(JavaUtilLoggerTest.class.getName()); logger.setLevel(Level.INFO); // When @@ -61,7 +64,7 @@ public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { @Test public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToFinest() { // Given - final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + final Logger logger = Logger.getLogger(JavaUtilLoggerTest.class.getName()); logger.setLevel(Level.FINEST); // When @@ -89,7 +92,7 @@ public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToFinest() { @Test public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabledEvenLoggingLogRecordsDirectly() { // Given - final Logger logger = Logger.getLogger(LoggerTest.class.getName()); + final Logger logger = Logger.getLogger(JavaUtilLoggerTest.class.getName()); logger.setLevel(Level.INFO); // When diff --git a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml index 4ae0e5b8b5..3d30136768 100644 --- a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml +++ b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml @@ -18,6 +18,16 @@ common: &default_settings wait_for_rpm_connect: false + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 2000 + local_decorating: + enabled: true + metrics: + enabled: true + # activemerchant instrumentation: only report dollar amounts (business data) # if the following is present and true; otherwise, only response times # and throughput to the gateway are reported From 771e2524bbb6a31d9c18bcb5233d100a70148713 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Feb 2022 16:32:12 -0800 Subject: [PATCH 48/96] Simplify instrumentation --- .../instrumentation/log4j1/AgentUtil.java | 28 +++---------------- .../instrumentation/log4j2/AgentUtil.java | 28 +++---------------- .../com/nr/instrumentation/jul/AgentUtil.java | 28 +++---------------- .../logbackclassic12/AgentUtil.java | 28 +++---------------- 4 files changed, 16 insertions(+), 96 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 2151cca046..fec877990e 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -71,12 +71,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); } /** @@ -85,12 +80,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); } /** @@ -99,12 +89,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); } /** @@ -113,11 +98,6 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index ffded56e1f..ef54d1ac52 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -106,12 +106,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); } /** @@ -120,12 +115,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); } /** @@ -134,12 +124,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); } /** @@ -148,11 +133,6 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } } diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 691ccf5e4f..e5c69c5a49 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -71,12 +71,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); } /** @@ -85,12 +80,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); } /** @@ -99,12 +89,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); } /** @@ -113,11 +98,6 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index d45dec8261..e238a5c99e 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -97,12 +97,7 @@ public static Map getFilteredLinkingMetadataMap() { * @return true if enabled, else false */ public static boolean isApplicationLoggingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); } /** @@ -111,12 +106,7 @@ public static boolean isApplicationLoggingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingMetricsEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); } /** @@ -125,12 +115,7 @@ public static boolean isApplicationLoggingMetricsEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingForwardingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); } /** @@ -139,11 +124,6 @@ public static boolean isApplicationLoggingForwardingEnabled() { * @return true if enabled, else false */ public static boolean isApplicationLoggingLocalDecoratingEnabled() { - Object configValue = NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - // Config value is parsed as a String if it was set by system property or environment variable - if (configValue instanceof String) { - return Boolean.parseBoolean((String) configValue); - } - return (Boolean) configValue; + return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } } From 93540aab73c7668a3fa5bf678cb9393f92914eb6 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Feb 2022 16:50:02 -0800 Subject: [PATCH 49/96] Fix instrumentation tests --- .../log4j1/Category_InstrumentationTest.java | 2 +- .../test/resources/application_logging_enabled.yml | 11 +++++++++++ .../log4j2/LoggerConfig_InstrumentationTest.java | 2 +- .../test/resources/application_logging_enabled.yml | 11 +++++++++++ .../logbackclassic12/Logger_InstrumentationTest.java | 2 +- .../test/resources/application_logging_enabled.yml | 11 +++++++++++ 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml create mode 100644 instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml create mode 100644 instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java index 9177cbd703..f1f8afb9dc 100644 --- a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java +++ b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/Category_InstrumentationTest.java @@ -10,7 +10,7 @@ import org.junit.runner.RunWith; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "org.apache.log4j" }) +@InstrumentationTestConfig(includePrefixes = { "org.apache.log4j" }, configName = "application_logging_enabled.yml") public class Category_InstrumentationTest { private static final String CAPTURED = "This log message should be captured"; diff --git a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml new file mode 100644 index 0000000000..4618c3d530 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml @@ -0,0 +1,11 @@ +common: &default_settings + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 2000 + local_decorating: + enabled: true + metrics: + enabled: true + diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index 137755b136..6747f48816 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -15,7 +15,7 @@ import org.junit.runner.RunWith; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "org.apache.logging.log4j.core" }) +@InstrumentationTestConfig(includePrefixes = { "org.apache.logging.log4j.core" }, configName = "application_logging_enabled.yml") public class LoggerConfig_InstrumentationTest extends TestCase { private static final String CAPTURED = "This log message should be captured"; diff --git a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml new file mode 100644 index 0000000000..4618c3d530 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml @@ -0,0 +1,11 @@ +common: &default_settings + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 2000 + local_decorating: + enabled: true + metrics: + enabled: true + diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java index 2fb6ae28bf..faa82aa20f 100644 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/Logger_InstrumentationTest.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; @RunWith(InstrumentationTestRunner.class) -@InstrumentationTestConfig(includePrefixes = { "ch.qos.logback" }) +@InstrumentationTestConfig(includePrefixes = { "ch.qos.logback" }, configName = "application_logging_enabled.yml") public class Logger_InstrumentationTest { private static final String CAPTURED = "This log message should be captured"; diff --git a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml new file mode 100644 index 0000000000..4618c3d530 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml @@ -0,0 +1,11 @@ +common: &default_settings + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 2000 + local_decorating: + enabled: true + metrics: + enabled: true + From 00d8698c3fba135295486cd66259090ada846b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Thu, 24 Feb 2022 15:39:24 +0100 Subject: [PATCH 50/96] fix: use formattedMessage when appending logback log messages --- .../logback/classic/spi/LoggingEvent_Instrumentation.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index e4253753a5..949c82592e 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -67,10 +67,14 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str if (applicationLoggingEnabled && isApplicationLoggingForwardingEnabled()) { // Record and send LogEvent to New Relic - recordNewRelicLogEvent(message, timeStamp, level); + recordNewRelicLogEvent(getFormattedMessage(), timeStamp, level); } } + public String getFormattedMessage() { + return Weaver.callOriginal(); + } + private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { return Weaver.callOriginal(); } From 87271f6ef9ffd3ddf26621daef1ee0722ac636c5 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 24 Feb 2022 15:52:18 -0800 Subject: [PATCH 51/96] Change default max_samples_stored to 10000. Set correct samples per harvest. --- .../src/test/resources/application_logging_enabled.yml | 2 +- .../src/test/resources/application_logging_enabled.yml | 2 +- .../src/test/resources/application_logging_enabled.yml | 2 +- .../src/main/java/com/newrelic/agent/HarvestServiceImpl.java | 4 +++- .../newrelic/agent/config/ApplicationLoggingConfigImpl.java | 2 +- .../agent/config/ApplicationLoggingForwardingConfig.java | 2 +- newrelic-agent/src/main/resources/newrelic.yml | 4 ++-- .../src/test/resources/com/newrelic/agent/config/newrelic.yml | 2 +- 8 files changed, 11 insertions(+), 9 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml index 4618c3d530..228169d1ba 100644 --- a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml @@ -3,7 +3,7 @@ common: &default_settings enabled: true forwarding: enabled: true - max_samples_stored: 2000 + max_samples_stored: 10000 local_decorating: enabled: true metrics: diff --git a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml index 4618c3d530..228169d1ba 100644 --- a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml @@ -3,7 +3,7 @@ common: &default_settings enabled: true forwarding: enabled: true - max_samples_stored: 2000 + max_samples_stored: 10000 local_decorating: enabled: true metrics: diff --git a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml index 4618c3d530..228169d1ba 100644 --- a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml @@ -3,7 +3,7 @@ common: &default_settings enabled: true forwarding: enabled: true - max_samples_stored: 2000 + max_samples_stored: 10000 local_decorating: enabled: true metrics: diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index 6ff67fbeb8..3f8ebc9245 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -108,7 +108,9 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { Long harvestLimit; // FIXME THIS IS A HACK! Real harvestLimit for log_event_data endpoint should come from server side event_harvest_config.harvest_limits!!! if (isLogEventEndpoint) { - int maxSamplesStoredLogEvents = ServiceFactory.getLogSenderService().getMaxSamplesStored(); + int logEventHarvestPeriodInSeconds = 5; + int harvestsPerMinute = 60 / logEventHarvestPeriodInSeconds; + int maxSamplesStoredLogEvents = ServiceFactory.getLogSenderService().getMaxSamplesStored() / harvestsPerMinute; harvestLimit = (long) maxSamplesStoredLogEvents; } else { harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java index 77cd1f9ab6..b89b60a955 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java @@ -18,7 +18,7 @@ * enabled: false * forwarding: * enabled: false - * max_samples_stored: 2000 + * max_samples_stored: 10000 * local_decorating: * enabled: false * metrics: diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java index edd1d42cfc..5e34240e17 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java @@ -15,7 +15,7 @@ public class ApplicationLoggingForwardingConfig extends BaseConfig { public static final String MAX_SAMPLES_STORED = "max_samples_stored"; public static final boolean DEFAULT_ENABLED = false; - public static final int DEFAULT_MAX_SAMPLES_STORED = 2000; + public static final int DEFAULT_MAX_SAMPLES_STORED = 10000; private final boolean enabled; private final int maxSamplesStored; diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 3bd277a36c..e7a709da5d 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -102,8 +102,8 @@ common: &default_settings # Application log events are collected up to the configured amount. Afterwards, # events are sampled to maintain an even distribution across the harvest cycle. - # Default is 2000. Setting to 0 will disable. - max_samples_stored: 2000 + # Default is 10000. Setting to 0 will disable. + max_samples_stored: 10000 # The agent will generate metrics to indicate the number of # application log events occurring at each distinct log level. diff --git a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml index 3d30136768..d1edea21f8 100644 --- a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml +++ b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml @@ -22,7 +22,7 @@ common: &default_settings enabled: true forwarding: enabled: true - max_samples_stored: 2000 + max_samples_stored: 10000 local_decorating: enabled: true metrics: From a024c9af591662ed1d9d8d8ee0a51c32f29bff8b Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 24 Feb 2022 16:38:28 -0800 Subject: [PATCH 52/96] Refactor default config values --- .../nr/agent/instrumentation/log4j1/AgentUtil.java | 4 ++-- .../test/resources/application_logging_enabled.yml | 4 ++-- .../nr/agent/instrumentation/log4j2/AgentUtil.java | 4 ++-- .../test/resources/application_logging_enabled.yml | 5 ++--- .../java/com/nr/instrumentation/jul/AgentUtil.java | 4 ++-- .../instrumentation/logbackclassic12/AgentUtil.java | 4 ++-- .../test/resources/application_logging_enabled.yml | 4 ++-- .../agent/config/ApplicationLoggingConfigImpl.java | 8 ++++---- .../config/ApplicationLoggingMetricsConfig.java | 2 +- newrelic-agent/src/main/resources/newrelic.yml | 12 ++++++------ .../resources/com/newrelic/agent/config/newrelic.yml | 4 ++-- 11 files changed, 27 insertions(+), 28 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index fec877990e..1c0a69a233 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -24,8 +24,8 @@ public class AgentUtil { private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; diff --git a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml index 228169d1ba..11348feea0 100644 --- a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml @@ -4,8 +4,8 @@ common: &default_settings forwarding: enabled: true max_samples_stored: 10000 - local_decorating: - enabled: true metrics: enabled: true + local_decorating: + enabled: true diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index ef54d1ac52..355f8dbb80 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -28,8 +28,8 @@ public class AgentUtil { private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; diff --git a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml index 228169d1ba..caabd2e331 100644 --- a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml @@ -4,8 +4,7 @@ common: &default_settings forwarding: enabled: true max_samples_stored: 10000 - local_decorating: - enabled: true metrics: enabled: true - + local_decorating: + enabled: true diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index e5c69c5a49..1a79f6e5f3 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -24,8 +24,8 @@ public class AgentUtil { private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index e238a5c99e..755f95df6d 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -26,8 +26,8 @@ public class AgentUtil { private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; + private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = false; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; diff --git a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml index 228169d1ba..11348feea0 100644 --- a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml @@ -4,8 +4,8 @@ common: &default_settings forwarding: enabled: true max_samples_stored: 10000 - local_decorating: - enabled: true metrics: enabled: true + local_decorating: + enabled: true diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java index b89b60a955..f7dbab0ae7 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java @@ -15,13 +15,13 @@ /* Default config should look like: * * application_logging: - * enabled: false + * enabled: true * forwarding: * enabled: false * max_samples_stored: 10000 - * local_decorating: - * enabled: false * metrics: + * enabled: true + * local_decorating: * enabled: false */ public class ApplicationLoggingConfigImpl extends BaseConfig implements ApplicationLoggingConfig { @@ -30,7 +30,7 @@ public class ApplicationLoggingConfigImpl extends BaseConfig implements Applicat public static final String FORWARDING = "forwarding"; public static final String LOCAL_DECORATING = "local_decorating"; - public static final boolean DEFAULT_ENABLED = false; + public static final boolean DEFAULT_ENABLED = true; public static final String ENABLED = "enabled"; private final ApplicationLoggingMetricsConfig applicationLoggingMetricsConfig; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java index 39a4aebdf3..25913d5760 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfig.java @@ -12,7 +12,7 @@ public class ApplicationLoggingMetricsConfig extends BaseConfig { public static final String ROOT = "metrics"; public static final String ENABLED = "enabled"; - public static final boolean DEFAULT_ENABLED = false; + public static final boolean DEFAULT_ENABLED = true; private final boolean enabled; diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index e7a709da5d..573432500c 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -88,10 +88,10 @@ common: &default_settings # The application_logging.forwarding and application_logging.local_decorating should not be used together. application_logging: - # Application logging features are disabled by default. Set this to true to enable - # control over the individual forwarding, local log decorating, and metrics features. - # Default is false. - enabled: false + # Provides control over all the application logging features for forwarding, local log + # decorating, and metrics features. Set as false to disable all application logging features. + # Default is true. + enabled: true # The agent will automatically forward application logs to New Relic in # a format that includes agent metadata for linking them to traces and errors. @@ -109,8 +109,8 @@ common: &default_settings # application log events occurring at each distinct log level. metrics: - # When true, application log metrics will be reported. The default is false. - enabled: false + # When true, application log metrics will be reported. The default is true. + enabled: true # The agent will add linking metadata to each log line in your application log files. # This feature should only be used if you want to use a third party log forwarder, instead diff --git a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml index d1edea21f8..710613719e 100644 --- a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml +++ b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml @@ -23,10 +23,10 @@ common: &default_settings forwarding: enabled: true max_samples_stored: 10000 - local_decorating: - enabled: true metrics: enabled: true + local_decorating: + enabled: true # activemerchant instrumentation: only report dollar amounts (business data) # if the following is present and true; otherwise, only response times From 6f45eec3c20287cda7a1ea418d4fb4c44f2bc6f7 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 25 Feb 2022 16:09:51 -0800 Subject: [PATCH 53/96] Add TODOs for tests --- .../test/java/com/newrelic/agent/model/LogEventTest.java | 7 +++++++ .../agent/introspec/internal/IntrospectorImpl.java | 2 ++ .../log4j2/LoggerConfig_InstrumentationTest.java | 3 +++ .../LoggingEvent_InstrumentationTest.java | 8 ++++++++ .../src/test/java/com/newrelic/agent/RPMServiceTest.java | 2 ++ .../newrelic/agent/attributes/AttributeValidatorTest.java | 2 ++ .../com/newrelic/agent/config/AgentConfigImplTest.java | 2 ++ .../agent/config/ApplicationLoggingConfigImplTest.java | 7 +++++++ .../config/ApplicationLoggingForwardingConfigTest.java | 7 +++++++ .../ApplicationLoggingLocalDecoratingConfigTest.java | 7 +++++++ .../agent/config/ApplicationLoggingMetricsConfigTest.java | 7 +++++++ .../agent/service/logging/LogSenderServiceImplTest.java | 7 +++++++ .../com/newrelic/agent/transport/DataSenderImplTest.java | 1 + 13 files changed, 62 insertions(+) create mode 100644 agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java create mode 100644 instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java diff --git a/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java b/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java new file mode 100644 index 0000000000..26fc0102e9 --- /dev/null +++ b/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.model; + +import static org.junit.Assert.*; + +public class LogEventTest { + // TODO +} \ No newline at end of file diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java index e058781a55..c9b9227248 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java @@ -228,6 +228,8 @@ public void clearSpanEvents() { service.clearReservoir(); } + // TODO add options to get/clear LogEvents to be used in instrumentation tests + @Override public int getRandomPort() { int port; diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index 6747f48816..2d29fcb71f 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -21,6 +21,9 @@ public class LoggerConfig_InstrumentationTest extends TestCase { private static final String CAPTURED = "This log message should be captured"; private static final String NOT_CAPTURED = "This message should NOT be captured"; + // TODO add tests for LogEvents being created when recordNewRelicLogEvent is called + // this is probably blocked until the Introspector is updated + @Test public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { // Given diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java new file mode 100644 index 0000000000..04e79e9b05 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java @@ -0,0 +1,8 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +import static org.junit.Assert.*; + +public class LoggingEvent_InstrumentationTest { + // TODO add tests for LogEvents being created when recordNewRelicLogEvent is called + // this is probably blocked until the Introspector is updated +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java index 80d3fdded5..f943d3d4ef 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java @@ -586,6 +586,8 @@ public void transactionNamingScheme() throws Exception { assertEquals(TransactionNamingScheme.LEGACY, svc.getTransactionNamingScheme()); } + // TODO sendLogEvents & sendLogEventsWithPut tests? + @Test(timeout = 30000) public void sendProfileData() throws Exception { Map map = createStagingMap(true, false); diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java index e0303f801a..026213cf84 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java @@ -186,6 +186,8 @@ public void testVerifyTruncatedValue() { assertEquals(expected, result); } + // TODO testVerifyTruncatedValue for LogEvent data + @Test public void testVerifySendOutsideTxn() { String methodCalled = "noticeError"; diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java index d8db9a6f24..d0cc5a3826 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java @@ -1223,6 +1223,8 @@ public void shouldSetSpanMaxSamplesWithServerProp() { assertEquals("Span max samples should be the harvest limit of: "+harvestLimit.intValue(), harvestLimit.intValue(), config.getSpanEventsConfig().getMaxSamplesStored()); } + // TODO test getApplicationLoggingConfig() + private static EnvironmentFacade createEnvironmentFacade( Map environment, Map systemProps) { EnvironmentFacade environmentFacade = new MapEnvironmentFacade(environment); diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java new file mode 100644 index 0000000000..09dbfb9a89 --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.config; + +import static org.junit.Assert.*; + +public class ApplicationLoggingConfigImplTest { + // TODO +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java new file mode 100644 index 0000000000..10735dffe4 --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.config; + +import static org.junit.Assert.*; + +public class ApplicationLoggingForwardingConfigTest { + // TODO +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java new file mode 100644 index 0000000000..e05061314e --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.config; + +import static org.junit.Assert.*; + +public class ApplicationLoggingLocalDecoratingConfigTest { + // TODO +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java new file mode 100644 index 0000000000..d36660f315 --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.config; + +import static org.junit.Assert.*; + +public class ApplicationLoggingMetricsConfigTest { + // TODO +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java new file mode 100644 index 0000000000..02dfd86803 --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java @@ -0,0 +1,7 @@ +package com.newrelic.agent.service.logging; + +import static org.junit.Assert.*; + +public class LogSenderServiceImplTest { + // TODO +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index 50c8477b4e..a232835419 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -531,6 +531,7 @@ public void testMaxPayloadSize() { sendAnalyticEventsPayloadTooBig(dataSender); sendMetricDataPayloadTooBig(dataSender); sendSpanEventsPayloadTooBig(dataSender); + // TODO sendLogEventsPayloadTooBig sendMetricDataSmallPayload(dataSender); From 988380e289170d4c930896b49895f6605c90fca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodr=C3=ADguez=20Gil?= Date: Mon, 28 Feb 2022 13:20:52 +0100 Subject: [PATCH 54/96] fix: log4j duplicated log lines and counters --- .../config/LoggerConfig_Instrumentation.java | 14 +++- .../LoggerConfig_InstrumentationTest.java | 82 ++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index 8c7171d7da..5e4b3f46e8 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -37,7 +37,8 @@ public LoggerConfig_Instrumentation() { protected void callAppenders(LogEvent event) { // Do nothing if application_logging.enabled: false - if (isApplicationLoggingEnabled()) { + // Do nothing if logger has parents and isAdditive is set to true to avoid duplicated counters and logs + if (isApplicationLoggingEnabled() && getParent() == null || !isAdditive()) { if (isApplicationLoggingMetricsEnabled()) { // Generate log level metrics NewRelic.incrementCounter("Logging/lines"); @@ -51,4 +52,13 @@ protected void callAppenders(LogEvent event) { } Weaver.callOriginal(); } -} \ No newline at end of file + + public LoggerConfig getParent() { + return Weaver.callOriginal(); + } + + public boolean isAdditive() { + return Weaver.callOriginal(); + } + +} diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index 2d29fcb71f..0f23e4a2d4 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -7,10 +7,17 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; +import org.apache.logging.log4j.core.config.AppenderRef; import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.layout.PatternLayout; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -24,6 +31,11 @@ public class LoggerConfig_InstrumentationTest extends TestCase { // TODO add tests for LogEvents being created when recordNewRelicLogEvent is called // this is probably blocked until the Introspector is updated + @Before + public void resetLoggerConfiguration() { + Configurator.reconfigure(); + } + @Test public void shouldIncrementEmittedLogsCountersIndependentlyIfLogLevelEnabled() { // Given @@ -73,6 +85,74 @@ public void shouldIncrementAllEmittedLogCountersIfLogLevelIsSetToTrace() { Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); } + @Test + public void shouldIncrementAllEmittedLogCountersOnlyOnceWhenMultipleLoggersAreSet() { + // Given + createLogger("A_SPECIAL_LOGGER", createAppender("ConsoleAppender"), Level.TRACE, true); + final Logger logger = LogManager.getLogger("A_SPECIAL_LOGGER"); + setLoggerLevel(Level.TRACE); + + // When + logger.trace(CAPTURED); + logger.debug(CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(5, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + @Test + public void shouldIncrementAllEmittedLogCountersRespectingLevelFromOriginalLogger() { + // Given + createLogger("A_SPECIAL_LOGGER", createAppender("ConsoleAppender"), Level.INFO, true); + final Logger logger = LogManager.getLogger("A_SPECIAL_LOGGER"); + setLoggerLevel(Level.ERROR); + + // When + logger.trace(NOT_CAPTURED); + logger.debug(NOT_CAPTURED); + logger.info(CAPTURED); + logger.warn(CAPTURED); + logger.error(CAPTURED); + + // Then + Assert.assertEquals(3, MetricsHelper.getUnscopedMetricCount("Logging/lines")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/TRACE")); + Assert.assertEquals(0, MetricsHelper.getUnscopedMetricCount("Logging/lines/DEBUG")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/INFO")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/WARN")); + Assert.assertEquals(1, MetricsHelper.getUnscopedMetricCount("Logging/lines/ERROR")); + } + + private void createLogger(String name, Appender appender, Level level, boolean additivity) { + final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + final Configuration config = ctx.getConfiguration(); + AppenderRef ref = AppenderRef.createAppenderRef("File", null, null); + AppenderRef[] refs = new AppenderRef[] {ref}; + LoggerConfig loggerConfig = LoggerConfig.createLogger(additivity, level, name, "true", refs, null, config, null ); + loggerConfig.addAppender(appender, level, null); + config.addLogger(name, loggerConfig); + } + + private Appender createAppender(String name) { + Layout layout = PatternLayout.newBuilder() + .withPattern(PatternLayout.SIMPLE_CONVERSION_PATTERN) + .build(); + Appender appender = ConsoleAppender.newBuilder() + .setName(name) + .setLayout(layout) + .build(); + appender.start(); + return appender; + } + private void setLoggerLevel(Level level) { final LoggerContext context = (LoggerContext) LogManager.getContext(false); final Configuration configuration = context.getConfiguration(); @@ -81,4 +161,4 @@ private void setLoggerLevel(Level level) { context.updateLoggers(); } -} \ No newline at end of file +} From dcfad8163cf5f9cba81ea6aca60dd452aef14e48 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 28 Feb 2022 08:53:49 -0800 Subject: [PATCH 55/96] Remove TODO --- .../newrelic/agent/service/logging/LogSenderServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 3d661691b1..2ced6af60d 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -403,7 +403,7 @@ public String getAppName() { } } catch (Exception e) { // discard harvest data - reservoir.clear(); // TODO should we simply discard all events if MaxPayloadException bubbles up to here from DataSenderImpl? + reservoir.clear(); Agent.LOG.log(Level.FINE, "Unable to send log events. Unsent events will be dropped.", e); } } From 67befaa8ec002920ec1d51779daab040fffde665 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 28 Feb 2022 15:56:59 -0800 Subject: [PATCH 56/96] Refactor metadata blob --- .../instrumentation/log4j1/AgentUtil.java | 34 ++++++++++++++++--- .../instrumentation/log4j2/AgentUtil.java | 34 ++++++++++++++++--- .../StringBuilderEncoder_Instrumentation.java | 4 +-- .../com/nr/instrumentation/jul/AgentUtil.java | 34 ++++++++++++++++--- .../spi/LoggingEvent_Instrumentation.java | 4 +-- .../logbackclassic12/AgentUtil.java | 34 ++++++++++++++++--- 6 files changed, 120 insertions(+), 24 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 1c0a69a233..0b5e068d9d 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -23,6 +23,13 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Linking metadata attributes used in blob + private static final String BLOB_PREFIX = "NR-LINKING"; + private static final String BLOB_DELIMITER = "|"; + private static final String TRACE_ID = "trace.id"; + private static final String HOSTNAME = "hostname"; + private static final String ENTITY_GUID = "entity.guid"; + private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; @@ -30,13 +37,30 @@ public class AgentUtil { private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** - * Gets a String representing the agent linking metadata after filtering - * out entity.type, entity.name, and any attributes with an empty value. + * Gets a String representing the agent linking metadata in blob format: + * NR-LINKING|entity.guid|hostname|trace.id|span.id| * - * @return Filtered String of agent linking metadata + * @return agent linking metadata string blob */ - public static String getFilteredLinkingMetadataString() { - return getFilteredLinkingMetadataMap().toString(); + public static String getLinkingMetadataBlob() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + StringBuilder blob = new StringBuilder(); + blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); + appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + } + return blob.toString(); + } + + private static void appendAttributeToBlob(String attribute, StringBuilder blob) { + if (attribute != null && !attribute.isEmpty()) { + blob.append(attribute); + } + blob.append(BLOB_DELIMITER); } /** diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 355f8dbb80..dbfcd4a8ad 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -27,6 +27,13 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Linking metadata attributes used in blob + private static final String BLOB_PREFIX = "NR-LINKING"; + private static final String BLOB_DELIMITER = "|"; + private static final String TRACE_ID = "trace.id"; + private static final String HOSTNAME = "hostname"; + private static final String ENTITY_GUID = "entity.guid"; + private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; @@ -65,13 +72,30 @@ public static void recordNewRelicLogEvent(LogEvent event) { } /** - * Gets a String representing the agent linking metadata after filtering - * out entity.type, entity.name, and any attributes with an empty value. + * Gets a String representing the agent linking metadata in blob format: + * NR-LINKING|entity.guid|hostname|trace.id|span.id| * - * @return Filtered String of agent linking metadata + * @return agent linking metadata string blob */ - public static String getFilteredLinkingMetadataString() { - return getFilteredLinkingMetadataMap().toString(); + public static String getLinkingMetadataBlob() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + StringBuilder blob = new StringBuilder(); + blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); + appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + } + return blob.toString(); + } + + private static void appendAttributeToBlob(String attribute, StringBuilder blob) { + if (attribute != null && !attribute.isEmpty()) { + blob.append(attribute); + } + blob.append(BLOB_DELIMITER); } /** diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java index 122c5f5c03..3a5810af76 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -11,7 +11,7 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.getFilteredLinkingMetadataString; +import static com.nr.agent.instrumentation.log4j2.AgentUtil.getLinkingMetadataBlob; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; @@ -34,7 +34,7 @@ private void appendAgentMetadata(StringBuilder source) { if (breakLine != -1) { source.replace(breakLine, breakLine + 1, ""); } - source.append(" NR-LINKING-METADATA: ").append(getFilteredLinkingMetadataString()).append("\n"); + source.append(getLinkingMetadataBlob()).append("\n"); } } \ No newline at end of file diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 1a79f6e5f3..e18c2d6b19 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -23,6 +23,13 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Linking metadata attributes used in blob + private static final String BLOB_PREFIX = "NR-LINKING"; + private static final String BLOB_DELIMITER = "|"; + private static final String TRACE_ID = "trace.id"; + private static final String HOSTNAME = "hostname"; + private static final String ENTITY_GUID = "entity.guid"; + private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; @@ -30,13 +37,30 @@ public class AgentUtil { private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; /** - * Gets a String representing the agent linking metadata after filtering - * out entity.type, entity.name, and any attributes with an empty value. + * Gets a String representing the agent linking metadata in blob format: + * NR-LINKING|entity.guid|hostname|trace.id|span.id| * - * @return Filtered String of agent linking metadata + * @return agent linking metadata string blob */ - public static String getFilteredLinkingMetadataString() { - return getFilteredLinkingMetadataMap().toString(); + public static String getLinkingMetadataBlob() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + StringBuilder blob = new StringBuilder(); + blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); + appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + } + return blob.toString(); + } + + private static void appendAttributeToBlob(String attribute, StringBuilder blob) { + if (attribute != null && !attribute.isEmpty()) { + blob.append(attribute); + } + blob.append(BLOB_DELIMITER); } /** diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index 949c82592e..15859cac00 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -14,7 +14,7 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getFilteredLinkingMetadataString; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getLinkingMetadataBlob; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingEnabled; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingForwardingEnabled; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; @@ -44,7 +44,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str boolean applicationLoggingEnabled = isApplicationLoggingEnabled(); if (applicationLoggingEnabled && isApplicationLoggingLocalDecoratingEnabled()) { // Append New Relic linking metadata from agent to log message - this.message = message + " NR-LINKING-METADATA: " + getFilteredLinkingMetadataString(); + this.message = message + getLinkingMetadataBlob(); } else { this.message = message; } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 755f95df6d..c1e9f635fb 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -25,6 +25,13 @@ public class AgentUtil { // Linking metadata attributes to filter out private static final String ENTITY_TYPE = "entity.type"; private static final String ENTITY_NAME = "entity.name"; + // Linking metadata attributes used in blob + private static final String BLOB_PREFIX = "NR-LINKING"; + private static final String BLOB_DELIMITER = "|"; + private static final String TRACE_ID = "trace.id"; + private static final String HOSTNAME = "hostname"; + private static final String ENTITY_GUID = "entity.guid"; + private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; @@ -56,13 +63,30 @@ public static void recordNewRelicLogEvent(String message, long timeStampMillis, } /** - * Gets a String representing the agent linking metadata after filtering - * out entity.type, entity.name, and any attributes with an empty value. + * Gets a String representing the agent linking metadata in blob format: + * NR-LINKING|entity.guid|hostname|trace.id|span.id| * - * @return Filtered String of agent linking metadata + * @return agent linking metadata string blob */ - public static String getFilteredLinkingMetadataString() { - return getFilteredLinkingMetadataMap().toString(); + public static String getLinkingMetadataBlob() { + Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); + StringBuilder blob = new StringBuilder(); + blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); + + if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); + appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + } + return blob.toString(); + } + + private static void appendAttributeToBlob(String attribute, StringBuilder blob) { + if (attribute != null && !attribute.isEmpty()) { + blob.append(attribute); + } + blob.append(BLOB_DELIMITER); } /** From 0c6da15576cec59364cf968a684b03153dcbb5de Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 2 Mar 2022 11:01:31 -0800 Subject: [PATCH 57/96] Update Introspector to test LogEvents --- .../agent/introspec/Introspector.java | 22 +++++++++++++++++++ .../introspec/internal/IntrospectorImpl.java | 13 ++++++++++- .../IntrospectorLogSenderService.java | 22 +++---------------- .../service/logging/LogSenderService.java | 2 +- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/Introspector.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/Introspector.java index 7d117ae669..8e6e928444 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/Introspector.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/Introspector.java @@ -7,6 +7,8 @@ package com.newrelic.agent.introspec; +import com.newrelic.agent.model.LogEvent; + import java.util.Collection; import java.util.Map; @@ -159,10 +161,30 @@ public interface Introspector { */ String getDispatcherVersion(); + /** + * Returns all span events that were collected since this was initialized or cleared. + * + * @return collection of SpanEvents or null if there are none + */ Collection getSpanEvents(); + /** + * Clear all existing SpanEvents + */ void clearSpanEvents(); + /** + * Returns all log events that were collected since this was initialized or cleared. + * + * @return collection of LogEvents or null if there are none + */ + Collection getLogEvents(); + + /** + * Clear all existing LogEvents + */ + void clearLogEvents(); + /** * Return random port available * diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java index c9b9227248..6d53183706 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorImpl.java @@ -22,6 +22,7 @@ import com.newrelic.agent.introspec.SpanEvent; import com.newrelic.agent.introspec.TracedMetricData; import com.newrelic.agent.introspec.TransactionTrace; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.service.ServiceFactory; import com.newrelic.agent.stats.TransactionStats; @@ -228,7 +229,17 @@ public void clearSpanEvents() { service.clearReservoir(); } - // TODO add options to get/clear LogEvents to be used in instrumentation tests + @Override + public Collection getLogEvents() { + IntrospectorLogSenderService service = (IntrospectorLogSenderService) ServiceFactory.getServiceManager().getLogSenderService(); + return service.getLogEvents(); + } + + @Override + public void clearLogEvents() { + IntrospectorLogSenderService service = (IntrospectorLogSenderService) ServiceFactory.getServiceManager().getLogSenderService(); + service.clearReservoir(); + } @Override public int getRandomPort() { diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java index e839f439d9..35392fccc1 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -11,21 +11,17 @@ import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.deps.com.google.common.collect.LinkedListMultimap; import com.newrelic.agent.deps.com.google.common.collect.ListMultimap; -import com.newrelic.agent.deps.com.google.common.collect.Multimaps; import com.newrelic.agent.deps.com.google.common.collect.Maps; -import com.newrelic.agent.introspec.Event; +import com.newrelic.agent.deps.com.google.common.collect.Multimaps; import com.newrelic.agent.logging.IAgentLogger; import com.newrelic.agent.model.AnalyticsEvent; import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.service.logging.LogSenderService; import com.newrelic.agent.tracing.DistributedTraceServiceImpl; -import com.newrelic.api.agent.Insights; import com.newrelic.api.agent.Logs; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.List; import java.util.Map; import static com.newrelic.agent.model.LogEvent.LOG_EVENT_TYPE; @@ -101,20 +97,8 @@ public void storeEvent(String appName, LogEvent event) { public void addHarvestableToService(String s) { } - public Collection getEventTypes() { - return Collections.unmodifiableCollection(events.keys()); - } - - public Collection getEvents(String type) { - List currentEvents = events.get(type); - if (currentEvents == null) { - return null; - } - List output = new ArrayList<>(currentEvents.size()); - for (LogEvent current : currentEvents) { - output.add(new EventImpl(current.getType(), current.getUserAttributesCopy())); - } - return output; + public Collection getLogEvents() { + return Collections.unmodifiableCollection(events.values()); } public void clear() { diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java index c31ceb8a14..2ad06b09ed 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderService.java @@ -35,7 +35,7 @@ public interface LogSenderService extends EventService, Logs { void storeEvent(String appName, LogEvent event); /** - * + * Register LogSenderHarvestable * @param appName application name */ void addHarvestableToService(String appName); From 3f2828a8627ded89cac9d05dc98032334cbd92dc Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 2 Mar 2022 18:34:57 -0800 Subject: [PATCH 58/96] Turn off local_decorating when running instrumentation tests --- .../src/test/resources/application_logging_enabled.yml | 3 +-- .../src/test/resources/application_logging_enabled.yml | 2 +- .../src/test/resources/application_logging_enabled.yml | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml index 11348feea0..583ffd2bba 100644 --- a/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-1/src/test/resources/application_logging_enabled.yml @@ -7,5 +7,4 @@ common: &default_settings metrics: enabled: true local_decorating: - enabled: true - + enabled: false diff --git a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml index caabd2e331..583ffd2bba 100644 --- a/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/apache-log4j-2/src/test/resources/application_logging_enabled.yml @@ -7,4 +7,4 @@ common: &default_settings metrics: enabled: true local_decorating: - enabled: true + enabled: false diff --git a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml index 11348feea0..583ffd2bba 100644 --- a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml +++ b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_enabled.yml @@ -7,5 +7,4 @@ common: &default_settings metrics: enabled: true local_decorating: - enabled: true - + enabled: false From 9ffd0fa81fb85f7c451daac35324b8b872bc018a Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 2 Mar 2022 18:35:51 -0800 Subject: [PATCH 59/96] Add instrumentation tests for creating LogEvents with Log4j2 --- .../LoggerConfig_InstrumentationTest.java | 459 +++++++++++++++++- 1 file changed, 448 insertions(+), 11 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index 0f23e4a2d4..df15051052 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -2,7 +2,9 @@ import com.newrelic.agent.introspec.InstrumentationTestConfig; import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.Introspector; import com.newrelic.agent.introspec.MetricsHelper; +import com.newrelic.agent.model.LogEvent; import junit.framework.TestCase; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -21,19 +23,454 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + @RunWith(InstrumentationTestRunner.class) @InstrumentationTestConfig(includePrefixes = { "org.apache.logging.log4j.core" }, configName = "application_logging_enabled.yml") public class LoggerConfig_InstrumentationTest extends TestCase { private static final String CAPTURED = "This log message should be captured"; private static final String NOT_CAPTURED = "This message should NOT be captured"; - - // TODO add tests for LogEvents being created when recordNewRelicLogEvent is called - // this is probably blocked until the Introspector is updated + private final Introspector introspector = InstrumentationTestRunner.getIntrospector(); @Before - public void resetLoggerConfiguration() { + public void reset() { Configurator.reconfigure(); + introspector.clearLogEvents(); + } + + @Test + public void testLogEventsAllLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at ALL level + setLoggerLevel(Level.ALL); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 2; + logger.error(CAPTURED); + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 3; + logger.warn(CAPTURED); + logger.warn(CAPTURED); + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 4; + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 5; + logger.debug(CAPTURED); + logger.debug(CAPTURED); + logger.debug(CAPTURED); + logger.debug(CAPTURED); + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 6; + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsOffLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Logging is OFF at all levels + setLoggerLevel(Level.OFF); + + int expectedTotalEventsCapturedAtFatal = 0; + logger.fatal(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtError = 0; + logger.error(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 0; + logger.warn(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = 0; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsFatalLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at FATAL level + setLoggerLevel(Level.FATAL); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 0; + logger.error(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 0; + logger.warn(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsErrorLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at ERROR level + setLoggerLevel(Level.ERROR); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 0; + logger.warn(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsWarnLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at WARN level + setLoggerLevel(Level.WARN); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsInfoLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at INFO level + setLoggerLevel(Level.INFO); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsDebugLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at DEBUG level + setLoggerLevel(Level.DEBUG); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 1; + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsTraceLevel() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + // Log at TRACE level + setLoggerLevel(Level.TRACE); + + int expectedTotalEventsCapturedAtFatal = 1; + logger.fatal(CAPTURED); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 1; + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 1; + logger.trace(CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtFatal + + expectedTotalEventsCapturedAtError + expectedTotalEventsCapturedAtWarn + + expectedTotalEventsCapturedAtInfo + expectedTotalEventsCapturedAtDebug + + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List fatalLevelLogEvents = getFatalLevelLogEvents(logEvents); + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtFatal, fatalLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + private List getTraceLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.TRACE.toString())) + .collect(Collectors.toList()); + } + + private List getDebugLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.DEBUG.toString())) + .collect(Collectors.toList()); + } + + private List getInfoLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.INFO.toString())) + .collect(Collectors.toList()); + } + + private List getWarnLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.WARN.toString())) + .collect(Collectors.toList()); + } + + private List getErrorLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.ERROR.toString())) + .collect(Collectors.toList()); + } + + private List getFatalLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.FATAL.toString())) + .collect(Collectors.toList()); } @Test @@ -135,20 +572,20 @@ private void createLogger(String name, Appender appender, Level level, boolean a final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); AppenderRef ref = AppenderRef.createAppenderRef("File", null, null); - AppenderRef[] refs = new AppenderRef[] {ref}; - LoggerConfig loggerConfig = LoggerConfig.createLogger(additivity, level, name, "true", refs, null, config, null ); + AppenderRef[] refs = new AppenderRef[] { ref }; + LoggerConfig loggerConfig = LoggerConfig.createLogger(additivity, level, name, "true", refs, null, config, null); loggerConfig.addAppender(appender, level, null); config.addLogger(name, loggerConfig); } private Appender createAppender(String name) { Layout layout = PatternLayout.newBuilder() - .withPattern(PatternLayout.SIMPLE_CONVERSION_PATTERN) - .build(); + .withPattern(PatternLayout.SIMPLE_CONVERSION_PATTERN) + .build(); Appender appender = ConsoleAppender.newBuilder() - .setName(name) - .setLayout(layout) - .build(); + .setName(name) + .setLayout(layout) + .build(); appender.start(); return appender; } From 7c49864a9716a0556c99670d45282aa0c667cebf Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Wed, 2 Mar 2022 09:39:42 -0500 Subject: [PATCH 60/96] Implement sendLogEvents tests in RPMServiceTest --- .../com/newrelic/agent/MockDataSender.java | 14 ++++-- .../com/newrelic/agent/RPMServiceTest.java | 45 ++++++++++++++++--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java index 6526581295..a38e911441 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java @@ -21,10 +21,7 @@ import com.newrelic.agent.transport.DataSenderListener; import org.json.simple.JSONStreamAware; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CountDownLatch; public class MockDataSender implements DataSender { @@ -35,6 +32,7 @@ public class MockDataSender implements DataSender { private static final String DATA_REPORT_PERIOD_KEY = "data_report_period"; private List traces; + private List logEvents; Map startupOptions; private Exception exception; private boolean isConnected; @@ -119,6 +117,10 @@ public List getTraces() { return traces; } + public List getLogEvents() { + return logEvents; + } + public Map getStartupOtions() { return startupOptions; } @@ -152,5 +154,9 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect @Override public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + if (exception != null) { + throw exception; + } + logEvents = new ArrayList<>(events); } } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java index f943d3d4ef..eb15982c3c 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java @@ -25,6 +25,7 @@ import com.newrelic.agent.errors.ErrorServiceImpl; import com.newrelic.agent.errors.ThrowableError; import com.newrelic.agent.metric.MetricName; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.normalization.NormalizationRule; import com.newrelic.agent.normalization.NormalizationRuleFactory; import com.newrelic.agent.profile.IProfile; @@ -87,10 +88,7 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -586,7 +584,44 @@ public void transactionNamingScheme() throws Exception { assertEquals(TransactionNamingScheme.LEGACY, svc.getTransactionNamingScheme()); } - // TODO sendLogEvents & sendLogEventsWithPut tests? + @Test(timeout = 30000) + public void testSendLogEvents() throws Exception { + Map map = createStagingMap(true, false); + map.put("app_name", "Test"); + AgentConfig config = AgentConfigImpl.createAgentConfig(map); + createServiceManager(config, map); + doSendLogEvent(); + } + + @Test(timeout = 30000) + public void testSendLogEventsWithPut() throws Exception { + Map map = createStagingMap(true, false, true); + map.put("app_name", "Test"); + AgentConfig config = AgentConfigImpl.createAgentConfig(map); + createServiceManager(config, map); + doSendLogEvent(); + } + + private void doSendLogEvent() throws Exception { + MockDataSenderFactory dataSenderFactory = new MockDataSenderFactory(); + DataSenderFactory.setDataSenderFactory(dataSenderFactory); + List appNames = singletonList("Send Log Events Test App"); + RPMService svc = new RPMService(appNames, null, null, Collections.emptyList()); + + svc.launch(); + + LogEvent logEvent1 = new LogEvent(null, 1); + LogEvent logEvent2 = new LogEvent(null, 2); + List logEvents = new ArrayList<>(); + logEvents.add(logEvent1); + logEvents.add(logEvent2); + + svc.sendLogEvents(0, 0, logEvents); + + List seen = dataSenderFactory.getLastDataSender().getLogEvents(); + + assertEquals("No log events sent currently", logEvents.size(), seen.size()); + } @Test(timeout = 30000) public void sendProfileData() throws Exception { From 9ba9b9529b9f93117503ede662e81de286745b5d Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 3 Mar 2022 10:50:12 -0800 Subject: [PATCH 61/96] Add instrumentation test for Logback 1.2 LogEvents --- .../LoggingEvent_InstrumentationTest.java | 361 +++++++++++++++++- 1 file changed, 358 insertions(+), 3 deletions(-) diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java index 04e79e9b05..4586af70a7 100644 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java @@ -1,8 +1,363 @@ package com.nr.agent.instrumentation.logbackclassic12; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.Introspector; +import com.newrelic.agent.model.LogEvent; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + import static org.junit.Assert.*; +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "ch.qos.logback" }, configName = "application_logging_enabled.yml") public class LoggingEvent_InstrumentationTest { - // TODO add tests for LogEvents being created when recordNewRelicLogEvent is called - // this is probably blocked until the Introspector is updated -} \ No newline at end of file + private static final String CAPTURED = "This log message should be captured"; + private static final String NOT_CAPTURED = "This message should NOT be captured"; + private final Introspector introspector = InstrumentationTestRunner.getIntrospector(); + + @Before + public void reset() { + introspector.clearLogEvents(); + } + + @Test + public void testLogEventsAllLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at ALL level + logger.setLevel(Level.ALL); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 2; + logger.warn(CAPTURED); + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 3; + logger.info(CAPTURED); + logger.info(CAPTURED); + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 4; + logger.debug(CAPTURED); + logger.debug(CAPTURED); + logger.debug(CAPTURED); + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 5; + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + logger.trace(CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsOffLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Logging is OFF at all levels + logger.setLevel(Level.OFF); + + int expectedTotalEventsCapturedAtError = 0; + logger.error(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 0; + logger.warn(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = 0; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsErrorLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at ERROR level + logger.setLevel(Level.ERROR); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 0; + logger.warn(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsWarnLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at WARN level + logger.setLevel(Level.WARN); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 0; + logger.info(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsInfoLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at INFO level + logger.setLevel(Level.INFO); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 0; + logger.debug(NOT_CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsDebugLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at DEBUG level + logger.setLevel(Level.DEBUG); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 1; + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 0; + logger.trace(NOT_CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + @Test + public void testLogEventsTraceLevel() { + final Logger logger = (Logger) LoggerFactory.getLogger(LoggingEvent_InstrumentationTest.class); + // Log at TRACE level + logger.setLevel(Level.TRACE); + + int expectedTotalEventsCapturedAtError = 1; + logger.error(CAPTURED); + + int expectedTotalEventsCapturedAtWarn = 1; + logger.warn(CAPTURED); + + int expectedTotalEventsCapturedAtInfo = 1; + logger.info(CAPTURED); + + int expectedTotalEventsCapturedAtDebug = 1; + logger.debug(CAPTURED); + + int expectedTotalEventsCapturedAtTrace = 1; + logger.trace(CAPTURED); + + int expectedTotalEventsCaptured = expectedTotalEventsCapturedAtError + + expectedTotalEventsCapturedAtWarn + expectedTotalEventsCapturedAtInfo + + expectedTotalEventsCapturedAtDebug + expectedTotalEventsCapturedAtTrace; + + Collection logEvents = introspector.getLogEvents(); + + assertEquals(expectedTotalEventsCaptured, logEvents.size()); + + List errorLevelLogEvents = getErrorLevelLogEvents(logEvents); + List warnLevelLogEvents = getWarnLevelLogEvents(logEvents); + List infoLevelLogEvents = getInfoLevelLogEvents(logEvents); + List debugLevelLogEvents = getDebugLevelLogEvents(logEvents); + List traceLevelLogEvents = getTraceLevelLogEvents(logEvents); + + assertEquals(expectedTotalEventsCapturedAtError, errorLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtWarn, warnLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtInfo, infoLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtDebug, debugLevelLogEvents.size()); + assertEquals(expectedTotalEventsCapturedAtTrace, traceLevelLogEvents.size()); + } + + private List getTraceLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.TRACE)) + .collect(Collectors.toList()); + } + + private List getDebugLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.DEBUG)) + .collect(Collectors.toList()); + } + + private List getInfoLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.INFO)) + .collect(Collectors.toList()); + } + + private List getWarnLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.WARN)) + .collect(Collectors.toList()); + } + + private List getErrorLevelLogEvents(Collection logEvents) { + return logEvents.stream() + .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.ERROR)) + .collect(Collectors.toList()); + } +} From 73264f5b508f8497463c665296a63ff700b55e6d Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 3 Mar 2022 16:16:53 -0800 Subject: [PATCH 62/96] Refactor instrumentation supportability metric --- .../logging/log4j/core/config/LoggerConfig_Instrumentation.java | 2 +- .../java/ch/qos/logback/classic/Logger_Instrumentation.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index 5e4b3f46e8..4359579be2 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -31,7 +31,7 @@ public class LoggerConfig_Instrumentation { public LoggerConfig_Instrumentation() { // Generate the instrumentation module supportability metric only once if (!instrumented.getAndSet(true)) { - NewRelic.incrementCounter("Supportability/Logging/enabled/Java/Log4j2"); + NewRelic.incrementCounter("Supportability/Logging/Java/Log4j2/enabled"); } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index e55878f657..b817526a65 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -27,7 +27,7 @@ public abstract class Logger_Instrumentation { Logger_Instrumentation() { // Generate the instrumentation module supportability metric only once if (!instrumented.getAndSet(true)) { - NewRelic.incrementCounter("Supportability/Logging/enabled/Java/LogbackClassic1.2"); + NewRelic.incrementCounter("Supportability/Logging/Java/LogbackClassic1.2/enabled"); } } From 9b9051df02f9200adb4c6fed2dad45fdf5994008 Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Fri, 4 Mar 2022 10:29:58 -0500 Subject: [PATCH 63/96] Implement LogEventTest --- .../newrelic/agent/model/LogEventTest.java | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java b/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java index 26fc0102e9..238fdac1e5 100644 --- a/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java +++ b/agent-model/src/test/java/com/newrelic/agent/model/LogEventTest.java @@ -1,7 +1,47 @@ package com.newrelic.agent.model; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.*; public class LogEventTest { - // TODO + + @Test + public void testConstructor() { + LogEvent logEvent = new LogEvent(Collections.emptyMap(), 0); + + assertEquals(logEvent.getPriority(), 0, 0); + assertNotNull(logEvent.getMutableUserAttributes()); + assertTrue(logEvent.getMutableUserAttributes().isEmpty()); + } + + @Test + public void testPriorityAccessors() { + LogEvent logEvent = new LogEvent(Collections.emptyMap(), 0); + + assertEquals(0, logEvent.getPriority(), 0); + + logEvent.setPriority(1); + + assertEquals(1, logEvent.getPriority(),0); + } + + @Test + public void testJsonString() throws IOException { + Map attributes = new HashMap<>(); + attributes.put("key", "value"); + + LogEvent logEvent = new LogEvent(attributes, 0); + StringWriter writer = new StringWriter(); + + logEvent.writeJSONString(writer); + + assertEquals("{\"key\":\"value\"}", writer.toString()); + } } \ No newline at end of file From 61cb6126dc3a2bca68272884d1c53237399e27d7 Mon Sep 17 00:00:00 2001 From: tbradellis Date: Thu, 3 Mar 2022 21:17:39 -0800 Subject: [PATCH 64/96] add tests for logging config --- .../agent/config/AgentConfigImplTest.java | 89 +++++++++++++++++-- 1 file changed, 80 insertions(+), 9 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java index d0cc5a3826..885e74b6e2 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/AgentConfigImplTest.java @@ -11,17 +11,26 @@ import com.newrelic.agent.config.internal.MapEnvironmentFacade; import com.newrelic.agent.config.internal.MapSystemProps; import com.newrelic.bootstrap.BootstrapAgent; - import org.junit.After; import org.junit.Test; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import static com.newrelic.agent.config.SpanEventsConfig.SERVER_SPAN_HARVEST_CONFIG; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /* (non-javadoc) - * Note: the "beacon" was a predecessor technology for correlated transaction traces with the browser. + * Note: the "beacon" was a predecessor technology for correlated transaction traces with the browser. * Some appearances of the term could be changed to "browser" now. */ @@ -92,7 +101,7 @@ public void collectorSetHost() { } @Test - public void defaultMetricIngestUri() { + public void defaultMetricIngestUri() { Map localMap = new HashMap<>(); // regular pre-protocol 15 key @@ -1120,7 +1129,7 @@ public void shouldLogPropertiesFromConfig() { List deprecationMessages = agentConfig.logDeprecatedProperties(settings); assertTrue(deprecationMessages.contains( - "Configuration my-prop is deprecated and will be removed in the next major version. " + "Configuration my-prop is deprecated and will be removed in the next major version. " + "It was set in the configuration file. " + "This property is obsolete." )); @@ -1143,7 +1152,6 @@ public void shouldLogPropertiesWithNewName() { )); } - @Test public void shouldLogPropertyAfterCheckingTheMap() { TestConfig target = new TestConfig(); @@ -1220,10 +1228,73 @@ public void shouldSetSpanMaxSamplesWithServerProp() { //when AgentConfig config = AgentConfigImpl.createAgentConfig(localMap); //then - assertEquals("Span max samples should be the harvest limit of: "+harvestLimit.intValue(), harvestLimit.intValue(), config.getSpanEventsConfig().getMaxSamplesStored()); + assertEquals("Span max samples should be the harvest limit of: " + harvestLimit.intValue(), harvestLimit.intValue(), + config.getSpanEventsConfig().getMaxSamplesStored()); } - // TODO test getApplicationLoggingConfig() + @Test + public void getApplicationLoggingConfig() { + long NOT_DEFAULT_MAX_SAMPLES_STORED = 5000; + Map subForwardingMap = new HashMap<>(); + subForwardingMap.put(ApplicationLoggingForwardingConfig.ENABLED, !ApplicationLoggingForwardingConfig.DEFAULT_ENABLED); + subForwardingMap.put(ApplicationLoggingForwardingConfig.MAX_SAMPLES_STORED, NOT_DEFAULT_MAX_SAMPLES_STORED); + + Map subMetricMap = new HashMap<>(); + subMetricMap.put(ApplicationLoggingMetricsConfig.ENABLED, !ApplicationLoggingMetricsConfig.DEFAULT_ENABLED); + + Map subDecoratingMap = new HashMap<>(); + subDecoratingMap.put(ApplicationLoggingLocalDecoratingConfig.ENABLED, !ApplicationLoggingLocalDecoratingConfig.DEFAULT_ENABLED); + + Map loggingMap = new HashMap<>(); + loggingMap.put(ApplicationLoggingConfigImpl.FORWARDING, subForwardingMap); + loggingMap.put(ApplicationLoggingConfigImpl.METRICS, subMetricMap); + loggingMap.put(ApplicationLoggingConfigImpl.LOCAL_DECORATING, subDecoratingMap); + + Map localMap = new HashMap<>(); + localMap.put(AgentConfigImpl.APPLICATION_LOGGING, loggingMap); + AgentConfig config = AgentConfigImpl.createAgentConfig(localMap); + + assertEquals(ApplicationLoggingConfigImpl.DEFAULT_ENABLED, config.getApplicationLoggingConfig().isEnabled()); + assertTrue(config.getApplicationLoggingConfig().isForwardingEnabled()); + assertEquals(NOT_DEFAULT_MAX_SAMPLES_STORED, config.getApplicationLoggingConfig().getMaxSamplesStored()); + assertFalse(config.getApplicationLoggingConfig().isMetricsEnabled()); + assertTrue(config.getApplicationLoggingConfig().isLocalDecoratingEnabled()); + + } + + @Test + public void getApplicationLoggingConfigDefaults() { + Map localMap = new HashMap<>(); + AgentConfig config = AgentConfigImpl.createAgentConfig(localMap); + + assertTrue(config.getApplicationLoggingConfig().isEnabled()); + assertFalse(config.getApplicationLoggingConfig().isForwardingEnabled()); + assertEquals(ApplicationLoggingForwardingConfig.DEFAULT_MAX_SAMPLES_STORED, config.getApplicationLoggingConfig().getMaxSamplesStored()); + assertTrue(config.getApplicationLoggingConfig().isMetricsEnabled()); + assertFalse(config.getApplicationLoggingConfig().isLocalDecoratingEnabled()); + + } + + @Test + public void getApplicationLoggingConfigSystemProperty() { + + String key = ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT + ApplicationLoggingConfigImpl.ENABLED; + String val = String.valueOf(!ApplicationLoggingConfigImpl.DEFAULT_ENABLED); + + Map properties = new HashMap<>(); + properties.put(key, val); + + SystemPropertyProvider provider = Mocks.createSystemPropertyProvider(properties); + + Map localMap = new HashMap<>(); + Map loggingMap = new HashMap<>(); + loggingMap.put(ApplicationLoggingConfigImpl.ENABLED, provider.getSystemProperty(key)); + localMap.put(ApplicationLoggingConfigImpl.FORWARDING, loggingMap); + AgentConfig config = AgentConfigImpl.createAgentConfig(localMap); + + assertFalse(ApplicationLoggingConfigImpl.DEFAULT_ENABLED && + config.getTransactionTracerConfig().isEnabled()); + } private static EnvironmentFacade createEnvironmentFacade( Map environment, Map systemProps) { From f76004fd664b8c6b5fa6416673bc50ff5e2b681e Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Fri, 4 Mar 2022 12:22:35 -0500 Subject: [PATCH 65/96] Add AttributeValidatorTest to verify LogEvent truncation --- .../attributes/AttributeValidatorTest.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java index 026213cf84..32e7e9ad4e 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/AttributeValidatorTest.java @@ -12,6 +12,7 @@ import com.newrelic.agent.MockServiceManager; import com.newrelic.agent.Transaction; import com.newrelic.agent.service.ServiceFactory; +import com.newrelic.agent.service.logging.LogSenderServiceImpl; import com.newrelic.agent.tracers.ClassMethodSignature; import com.newrelic.agent.tracers.servlet.BasicRequestRootTracer; import com.newrelic.agent.tracers.servlet.MockHttpRequest; @@ -186,7 +187,24 @@ public void testVerifyTruncatedValue() { assertEquals(expected, result); } - // TODO testVerifyTruncatedValue for LogEvent data + @Test + public void testVerifyTruncatedValueForLogEventData() { + Map input = new HashMap<>(); + String longValue = Strings.padEnd("", 33000, 'e'); + String longExpectedValue = Strings.padEnd("", 32767, 'e'); + input.put("key", longValue); + input.put("apple", "pie"); + input.put("sugar", "cream"); + + Map expected = ImmutableMap.of("apple", "pie", "sugar", "cream", "key", longExpectedValue); + + AttributeValidator attributeValidator = new AttributeValidator(ATTRIBUTE_TYPE); + + attributeValidator.setTransactional(false); + Map result = attributeValidator.verifyParametersAndReturnValues(input, LogSenderServiceImpl.METHOD); + + assertEquals(expected, result); + } @Test public void testVerifySendOutsideTxn() { From e84c83a9c7041f57cb7ffb416c573935c141b74c Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Fri, 4 Mar 2022 10:39:43 -0800 Subject: [PATCH 66/96] Implement event_harvest_config.harvest_limits for log_event_data --- .../com/newrelic/agent/HarvestServiceImpl.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index 3f8ebc9245..3cf62666fe 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -85,7 +85,7 @@ public void startHarvest(IRPMService rpmService) { public void startHarvestables(IRPMService rpmService, AgentConfig config) { Map eventHarvestConfig = config.getProperty(AgentConfigFactory.EVENT_HARVEST_CONFIG); Map spanHarvestConfig = config.getProperty(SERVER_SPAN_HARVEST_CONFIG); - // FIXME need to add log event harvest config when it becomes available from the backend + if (eventHarvestConfig == null) { ServiceFactory.getStatsService().doStatsWork(StatsWorks.getIncrementCounterWork( MetricNames.SUPPORTABILITY_CONNECT_MISSING_EVENT_DATA, 1)); @@ -96,25 +96,15 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { int maxSamplesStored = tracker.harvestable.getMaxSamplesStored(); long reportPeriodInMillis = HarvestServiceImpl.REPORTING_PERIOD_IN_MILLISECONDS; boolean isSpanEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(SPAN_EVENT_DATA); - boolean isLogEventEndpoint = tracker.harvestable.getEndpointMethodName().equals(LOG_EVENT_DATA); - // The event_harvest_config.harvest_limits received from server-side during the connect lifecycle contains config for error_event_data, - // analytic_event_data, and custom_event_data TODO The event_harvest_config.harvest_limits should also include log_event_data endpoint at some point... + // The event_harvest_config.harvest_limits received from server-side during the connect lifecycle + // contains config for error_event_data, analytic_event_data, custom_event_data, and log_event_data if (eventHarvestConfig != null && !isSpanEventEndpoint) { Agent.LOG.log(Level.FINE, "event_harvest_config from collector is: {0} samples stored for {1}", maxSamplesStored, tracker.harvestable.getEndpointMethodName()); Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); - Long harvestLimit; - // FIXME THIS IS A HACK! Real harvestLimit for log_event_data endpoint should come from server side event_harvest_config.harvest_limits!!! - if (isLogEventEndpoint) { - int logEventHarvestPeriodInSeconds = 5; - int harvestsPerMinute = 60 / logEventHarvestPeriodInSeconds; - int maxSamplesStoredLogEvents = ServiceFactory.getLogSenderService().getMaxSamplesStored() / harvestsPerMinute; - harvestLimit = (long) maxSamplesStoredLogEvents; - } else { - harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); - } + Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); if (harvestLimit != null) { maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period From 8a77aeed417c3dbedc6665f32c005f657d13f994 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 7 Mar 2022 12:43:09 -0800 Subject: [PATCH 67/96] Fix doStatsWork compilation failure --- .../newrelic/agent/service/logging/LogSenderServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 2ced6af60d..6a98195b5e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -384,7 +384,7 @@ public void doWork(StatsEngine statsEngine) { public String getAppName() { return appName; } - }); + }, LogSenderServiceImpl.class.getName()); if (reservoir.size() < reservoir.getNumberOfTries()) { int dropped = reservoir.getNumberOfTries() - reservoir.size(); From c278636605015c5aa2bb363b9435cba137785201 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 7 Mar 2022 15:48:07 -0800 Subject: [PATCH 68/96] Safeguard against NPE --- .../instrumentation/log4j2/AgentUtil.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index dbfcd4a8ad..abf04e49e0 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -48,25 +48,26 @@ public class AgentUtil { public static void recordNewRelicLogEvent(LogEvent event) { if (event != null) { Message message = event.getMessage(); - String formattedMessage = message.getFormattedMessage(); + if (message != null) { + String formattedMessage = message.getFormattedMessage(); + // Bail out and don't create a LogEvent if log message is empty + if (formattedMessage != null && !formattedMessage.isEmpty()) { + HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); - // Bail out and don't create a LogEvent if log message is empty - if (!formattedMessage.isEmpty()) { - HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); + logEventMap.put(MESSAGE, formattedMessage); + logEventMap.put(TIMESTAMP, event.getTimeMillis()); - logEventMap.put(MESSAGE, formattedMessage); - logEventMap.put(TIMESTAMP, event.getTimeMillis()); + Level level = event.getLevel(); + String levelName = level.name(); - Level level = event.getLevel(); - String levelName = level.name(); + if (levelName.isEmpty()) { + logEventMap.put(LOG_LEVEL, UNKNOWN); + } else { + logEventMap.put(LOG_LEVEL, levelName); + } - if (levelName.isEmpty()) { - logEventMap.put(LOG_LEVEL, UNKNOWN); - } else { - logEventMap.put(LOG_LEVEL, levelName); + AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } - - AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); } } } From 2b0c84215f6a4a2aa44338543b28238e9ea373a9 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 7 Mar 2022 15:52:05 -0800 Subject: [PATCH 69/96] Safeguard against NPE --- .../nr/agent/instrumentation/log4j2/AgentUtil.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index abf04e49e0..a58156de32 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -58,12 +58,13 @@ public static void recordNewRelicLogEvent(LogEvent event) { logEventMap.put(TIMESTAMP, event.getTimeMillis()); Level level = event.getLevel(); - String levelName = level.name(); - - if (levelName.isEmpty()) { - logEventMap.put(LOG_LEVEL, UNKNOWN); - } else { - logEventMap.put(LOG_LEVEL, levelName); + if (level != null) { + String levelName = level.name(); + if (levelName.isEmpty()) { + logEventMap.put(LOG_LEVEL, UNKNOWN); + } else { + logEventMap.put(LOG_LEVEL, levelName); + } } AgentBridge.getAgent().getLogSender().recordLogEvent(logEventMap); From 28ec836cfc126d13f22ac82f69f6bae539e3eefd Mon Sep 17 00:00:00 2001 From: tbradellis Date: Mon, 7 Mar 2022 21:49:29 -0800 Subject: [PATCH 70/96] adds tests for AppLoggingConfig --- .../ApplicationLoggingConfigImplTest.java | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java index 09dbfb9a89..4b608e517f 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingConfigImplTest.java @@ -1,7 +1,76 @@ package com.newrelic.agent.config; -import static org.junit.Assert.*; +import com.newrelic.agent.SaveSystemPropertyProviderRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class ApplicationLoggingConfigImplTest { - // TODO + + private Map localProps; + + @Rule + public SaveSystemPropertyProviderRule saveSystemPropertyProviderRule = new SaveSystemPropertyProviderRule(); + + @Before + public void setup() { + localProps = new HashMap<>(); + } + + @Test + public void testShouldBeEnabled() { + ApplicationLoggingConfigImpl config = new ApplicationLoggingConfigImpl(localProps, false); + assertTrue(config.isEnabled()); + } + + @Test + public void testDisabledOrNotWithHighSecurity() { + ApplicationLoggingConfigImpl config = new ApplicationLoggingConfigImpl(localProps, true); + assertTrue(config.isEnabled()); + assertTrue(config.isMetricsEnabled()); + + assertFalse(config.isLocalDecoratingEnabled()); + assertFalse(config.isForwardingEnabled()); + + } + + @Test + public void canConfigureViaSystemProperty() { + Properties properties = new Properties(); + //default application_logging is true + properties.put("newrelic.config.application_logging.enabled", "false"); + + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(properties), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade(Collections.emptyMap()) //only test the system property + + )); + + ApplicationLoggingConfigImpl config = new ApplicationLoggingConfigImpl(Collections.emptyMap(), false); + assertFalse(config.isEnabled()); + + } + + @Test + public void canConfigureViaEnvironmentVariables() { + + //default forwarding is false + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(), //use default configs except for forwarding environment variable + new SaveSystemPropertyProviderRule.TestEnvironmentFacade(Collections.singletonMap("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", "true")) + )); + + ApplicationLoggingConfigImpl config = new ApplicationLoggingConfigImpl(Collections.emptyMap(), false); + + assertTrue(config.isForwardingEnabled()); + + } } \ No newline at end of file From d98d3559b8a129f7858fb422388a5dc22f761a3e Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Mon, 7 Mar 2022 11:22:32 -0500 Subject: [PATCH 71/96] Add sendLogEventsPayloadToBig tests * add tests to DataSenderImplTest.testMaxPayloadSize() --- .../agent/transport/DataSenderImpl.java | 2 +- .../agent/transport/DataSenderImplTest.java | 36 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index ff6e68748c..8a9ebda4d1 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -570,7 +570,7 @@ private ReadResult connectAndSend(String host, String method, String encoding, S byte[] data = writeData(encoding, params); /* - * We don't enforce max_payload_size_in_bytes for error_data (aka error traces). Instead we halve the + * We don't enforce max_payload_size_in_bytes for error_data (aka error traces). Instead, we halve the * payload and try again. See RPMService sendErrorData */ if (data.length > maxPayloadSizeInBytes && !method.equals(CollectorMethods.ERROR_DATA)) { diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index a232835419..ae5da250b3 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -19,6 +19,7 @@ import com.newrelic.agent.config.AgentConfigImpl; import com.newrelic.agent.logging.IAgentLogger; import com.newrelic.agent.metric.MetricName; +import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.model.PathHashes; import com.newrelic.agent.model.SpanCategory; import com.newrelic.agent.model.SpanEvent; @@ -50,10 +51,7 @@ import java.net.URL; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import static com.newrelic.agent.MetricNames.SUPPORTABILITY_AGENT_ENDPOINT_HTTP_ERROR; @@ -70,6 +68,7 @@ public class DataSenderImplTest { private static final String SUPPORTABILITY_METRIC_METRIC_DATA = "Supportability/Agent/Collector/MaxPayloadSizeLimit/metric_data"; private static final String SUPPORTABILITY_METRIC_ANALYTIC_DATA = "Supportability/Agent/Collector/MaxPayloadSizeLimit/analytic_event_data"; private static final String SUPPORTABILITY_METRIC_SPAN_DATA = "Supportability/Agent/Collector/MaxPayloadSizeLimit/span_event_data"; + private static final String MAX_PAYLOAD_EXCEPTION = MaxPayloadException.class.getSimpleName(); @Rule public ExpectedException exceptionRule = ExpectedException.none(); @@ -531,7 +530,7 @@ public void testMaxPayloadSize() { sendAnalyticEventsPayloadTooBig(dataSender); sendMetricDataPayloadTooBig(dataSender); sendSpanEventsPayloadTooBig(dataSender); - // TODO sendLogEventsPayloadTooBig + sendLogEventsPayloadTooBig(dataSender); sendMetricDataSmallPayload(dataSender); @@ -597,12 +596,23 @@ private void sendMetricDataSmallPayload(DataSenderImpl dataSender) { } } + private void sendLogEventsPayloadTooBig(DataSenderImpl dataSender) { + boolean exceptionThrown = false; + try { + dataSender.sendLogEvents(10000, 10000, createLogEvents(10000)); + } catch (Exception e) { + assertEquals(MAX_PAYLOAD_EXCEPTION, e.getClass().getSimpleName()); + exceptionThrown = true; + } + assertTrue("MaxPayloadException was NOT thrown as expected", exceptionThrown); + } + private void sendAnalyticEventsPayloadTooBig(DataSenderImpl dataSender) { try { // ~ 943 bytes dataSender.sendAnalyticsEvents(10000, 10000, createTransactionEvents(1000)); } catch (Exception e) { - assertEquals("MaxPayloadException", e.getClass().getSimpleName()); + assertEquals(MAX_PAYLOAD_EXCEPTION, e.getClass().getSimpleName()); } } @@ -611,7 +621,7 @@ private void sendMetricDataPayloadTooBig(DataSenderImpl dataSender) { // ~ 2378 bytes dataSender.sendMetricData(System.currentTimeMillis() - 60, System.currentTimeMillis(), createMetricData(1000)); } catch (Exception e) { - assertEquals("MaxPayloadException", e.getClass().getSimpleName()); + assertEquals(MAX_PAYLOAD_EXCEPTION, e.getClass().getSimpleName()); } } @@ -620,7 +630,7 @@ private void sendSpanEventsPayloadTooBig(DataSenderImpl dataSender) { // ~ 999 bytes dataSender.sendSpanEvents(10000, 10000, createSpanEvents(1000)); } catch (Exception e) { - assertEquals("MaxPayloadException", e.getClass().getSimpleName()); + assertEquals(MAX_PAYLOAD_EXCEPTION, e.getClass().getSimpleName()); } } @@ -654,6 +664,16 @@ private List createTransactionEvents(int size) { return events; } + private List createLogEvents(int size) { + List events = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Map attrs = new HashMap<>(); + attrs.put("key", "value"); + events.add(new LogEvent(attrs, 0)); + } + return events; + } + private List createMetricData(int metrics) { List metricData = new ArrayList<>(); for (int i = 0; i < metrics; i++) { From fa8efcbcb34712698a324ceaec0b26efa547b9d8 Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Tue, 8 Mar 2022 11:18:03 -0500 Subject: [PATCH 72/96] Refactor send log events to remove unused parameters --- .../introspec/internal/IntrospectorRPMService.java | 2 +- .../src/main/java/com/newrelic/agent/IRPMService.java | 2 +- .../src/main/java/com/newrelic/agent/RPMService.java | 10 +++++----- .../agent/service/logging/LogSenderServiceImpl.java | 2 +- .../java/com/newrelic/agent/transport/DataSender.java | 2 +- .../com/newrelic/agent/transport/DataSenderImpl.java | 2 +- .../test/java/com/newrelic/agent/BaseRPMService.java | 2 +- .../test/java/com/newrelic/agent/MockDataSender.java | 2 +- .../test/java/com/newrelic/agent/MockRPMService.java | 4 ++-- .../test/java/com/newrelic/agent/RPMServiceTest.java | 2 +- .../newrelic/agent/transport/DataSenderImplTest.java | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java index 7ac384393f..b0cc1659ac 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorRPMService.java @@ -151,7 +151,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) { + public void sendLogEvents(Collection events) { } @Override diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java index 7fa6a8150c..aecb050f72 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/IRPMService.java @@ -83,7 +83,7 @@ public interface IRPMService extends Service { void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; - void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + void sendLogEvents(Collection events) throws Exception; void sendErrorEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception; diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java index 66c464a13c..07cd45dd04 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/RPMService.java @@ -532,10 +532,10 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, final C } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, final Collection events) throws Exception { + public void sendLogEvents(final Collection events) throws Exception { Agent.LOG.log(Level.FINE, "Sending {0} log event(s)", events.size()); try { - sendLogEventsSyncRestart(reservoirSize, eventsSeen, events); + sendLogEventsSyncRestart(events); } catch (HttpError e) { // We don't want to resend the data for certain response codes, retry for all others if (e.isRetryableError()) { @@ -594,14 +594,14 @@ private void sendCustomAnalyticsEventsSyncRestart(int reservoirSize, int eventsS } } - private void sendLogEventsSyncRestart(int reservoirSize, int eventsSeen, final Collection events) + private void sendLogEventsSyncRestart(final Collection events) throws Exception { try { - dataSender.sendLogEvents(reservoirSize, eventsSeen, events); + dataSender.sendLogEvents(events); } catch (ForceRestartException e) { logForceRestartException(e); reconnectSync(); - dataSender.sendLogEvents(reservoirSize, eventsSeen, events); + dataSender.sendLogEvents(events); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 6a98195b5e..462417077e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -371,7 +371,7 @@ public void harvestEvents(final String appName) { // Send LogEvents ServiceFactory.getRPMServiceManager() .getOrCreateRPMService(appName) - .sendLogEvents(maxSamplesStored, reservoir.getNumberOfTries(), Collections.unmodifiableList(reservoir.asList())); + .sendLogEvents(Collections.unmodifiableList(reservoir.asList())); final long durationInNanos = System.nanoTime() - startTimeInNanos; ServiceFactory.getStatsService().doStatsWork(new StatsWork() { diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java index 2b0484d958..165f2ffd72 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSender.java @@ -48,7 +48,7 @@ public interface DataSender { /** * Send non-aggregated Log events */ - void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception; + void sendLogEvents(Collection events) throws Exception; /** * Send non-aggregated span events diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java index 8a9ebda4d1..46ff8b7e1e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/transport/DataSenderImpl.java @@ -336,7 +336,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + public void sendLogEvents(Collection events) throws Exception { sendLogEventsForReservoir(CollectorMethods.LOG_EVENT_DATA, compressedEncoding, events); } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java b/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java index 081a59b236..5f7aeb1ba1 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/BaseRPMService.java @@ -164,7 +164,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + public void sendLogEvents(Collection events) throws Exception { } @Override diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java index a38e911441..9c461c71f6 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockDataSender.java @@ -153,7 +153,7 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + public void sendLogEvents(Collection events) throws Exception { if (exception != null) { throw exception; } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java b/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java index 66e30f6b5d..b8e3244af0 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/MockRPMService.java @@ -225,9 +225,9 @@ public void sendCustomAnalyticsEvents(int reservoirSize, int eventsSeen, Collect } @Override - public void sendLogEvents(int reservoirSize, int eventsSeen, Collection events) throws Exception { + public void sendLogEvents(Collection events) throws Exception { this.events.addAll(events); - this.logSenderEventsSeen.addAndGet(eventsSeen); + this.logSenderEventsSeen.addAndGet(events.size()); } @Override diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java index eb15982c3c..d630f9db4f 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/RPMServiceTest.java @@ -616,7 +616,7 @@ private void doSendLogEvent() throws Exception { logEvents.add(logEvent1); logEvents.add(logEvent2); - svc.sendLogEvents(0, 0, logEvents); + svc.sendLogEvents(logEvents); List seen = dataSenderFactory.getLastDataSender().getLogEvents(); diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index ae5da250b3..f5c7ee47c2 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -599,7 +599,7 @@ private void sendMetricDataSmallPayload(DataSenderImpl dataSender) { private void sendLogEventsPayloadTooBig(DataSenderImpl dataSender) { boolean exceptionThrown = false; try { - dataSender.sendLogEvents(10000, 10000, createLogEvents(10000)); + dataSender.sendLogEvents(createLogEvents(10000)); } catch (Exception e) { assertEquals(MAX_PAYLOAD_EXCEPTION, e.getClass().getSimpleName()); exceptionThrown = true; From 25c85b5c7fac4838cc21dd1cfb31269aa6d58859 Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Tue, 8 Mar 2022 16:01:32 -0500 Subject: [PATCH 73/96] Fix imports via code style --- .../com/newrelic/agent/transport/DataSenderImplTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java index f5c7ee47c2..49ae1b3576 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/transport/DataSenderImplTest.java @@ -51,7 +51,10 @@ import java.net.URL; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static com.newrelic.agent.MetricNames.SUPPORTABILITY_AGENT_ENDPOINT_HTTP_ERROR; From ee135a21ad19ac5d97a212dc0a4fbb9ed4623614 Mon Sep 17 00:00:00 2001 From: tbradellis Date: Tue, 8 Mar 2022 20:32:24 -0800 Subject: [PATCH 74/96] log forwarding tests --- ...pplicationLoggingForwardingConfigTest.java | 81 ++++++++++++++++++- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java index 10735dffe4..745e7ff436 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java @@ -1,7 +1,82 @@ package com.newrelic.agent.config; -import static org.junit.Assert.*; +import com.newrelic.agent.SaveSystemPropertyProviderRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; public class ApplicationLoggingForwardingConfigTest { - // TODO -} \ No newline at end of file + private Map localProps; + private static final int TEST_MAX_SAMPLES_STORED = 5000; + + @Rule + public SaveSystemPropertyProviderRule saveSystemPropertyProviderRule = new SaveSystemPropertyProviderRule(); + + @Before + public void setup() { + localProps = new HashMap<>(); + } + + @Test + public void defaultForwardingConfig() { + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + false); + assertFalse(config.getEnabled()); + + } + + @Test + public void testMaxSamplesStoredDefaultValue() { + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + false); + assertEquals(ApplicationLoggingForwardingConfig.DEFAULT_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); + } + + @Test + public void testMaxSamplesStoredNotDefaultValue() { + localProps.put(ApplicationLoggingForwardingConfig.MAX_SAMPLES_STORED, TEST_MAX_SAMPLES_STORED); + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + false); + assertEquals(TEST_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); + } + + @Test + public void usesEnvVarForNestedConfig() { + + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade( + Collections.singletonMap("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_MAX_SAMPLES_STORED", "5000")) + )); + + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, false); + assertEquals(5000, config.getMaxSamplesStored()); + + } + + @Test + public void usesSysPropForNestedConfig() { + Properties properties = new Properties(); + + properties.put("newrelic.config.application_logging.forwarding.max_samples_stored", "" + TEST_MAX_SAMPLES_STORED); + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(properties), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade() + )); + + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, false); + assertEquals(5000, config.getMaxSamplesStored()); + + } + +} From ac7d81981e691efdede0c9810fd0aade4500a73c Mon Sep 17 00:00:00 2001 From: tbradellis Date: Thu, 10 Mar 2022 13:22:41 -0800 Subject: [PATCH 75/96] clean up test max samples --- .../agent/config/ApplicationLoggingForwardingConfigTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java index 745e7ff436..cf59b54d20 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java @@ -59,7 +59,7 @@ public void usesEnvVarForNestedConfig() { ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(Collections.emptyMap(), ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, false); - assertEquals(5000, config.getMaxSamplesStored()); + assertEquals(TEST_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); } @@ -75,7 +75,7 @@ public void usesSysPropForNestedConfig() { ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(Collections.emptyMap(), ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, false); - assertEquals(5000, config.getMaxSamplesStored()); + assertEquals(TEST_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); } From 98b5576eae105ff82fce46ea8153b6e73454bfc2 Mon Sep 17 00:00:00 2001 From: tbradellis Date: Wed, 9 Mar 2022 13:03:33 -0800 Subject: [PATCH 76/96] add logging metric tests --- .../ApplicationLoggingMetricsConfigTest.java | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java index d36660f315..bac18dc8c0 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java @@ -1,7 +1,65 @@ package com.newrelic.agent.config; +import com.newrelic.agent.SaveSystemPropertyProviderRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + import static org.junit.Assert.*; public class ApplicationLoggingMetricsConfigTest { - // TODO + + private Map localProps; + + @Rule + public SaveSystemPropertyProviderRule saveSystemPropertyProviderRule = new SaveSystemPropertyProviderRule(); + + @Before + public void setup() { + localProps = new HashMap<>(); + } + + @Test + public void defaultLocalDecoratingConfig() { + ApplicationLoggingMetricsConfig config = new ApplicationLoggingMetricsConfig(localProps, + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertTrue(config.getEnabled()); + + } + + @Test + public void usesEnvVarForLocalDecoratingConfig() { + + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade( + Collections.singletonMap("NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED", "false")) + )); + + ApplicationLoggingMetricsConfig config = new ApplicationLoggingMetricsConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertFalse(config.getEnabled()); + + } + + @Test + public void usesSysPropForLocalDecoratingConfig() { + Properties properties = new Properties(); + + properties.put("newrelic.config.application_logging.metrics.enabled", "" + "false"); + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(properties), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade() + )); + + ApplicationLoggingMetricsConfig config = new ApplicationLoggingMetricsConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertFalse(config.getEnabled()); + + } } \ No newline at end of file From def0751eb18ba3476251edc5f26605ff7b185041 Mon Sep 17 00:00:00 2001 From: tbradellis Date: Thu, 10 Mar 2022 13:24:01 -0800 Subject: [PATCH 77/96] fix string concat --- .../agent/config/ApplicationLoggingMetricsConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java index bac18dc8c0..28f771ddea 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingMetricsConfigTest.java @@ -51,7 +51,7 @@ public void usesEnvVarForLocalDecoratingConfig() { public void usesSysPropForLocalDecoratingConfig() { Properties properties = new Properties(); - properties.put("newrelic.config.application_logging.metrics.enabled", "" + "false"); + properties.put("newrelic.config.application_logging.metrics.enabled", "false"); SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( new SaveSystemPropertyProviderRule.TestSystemProps(properties), new SaveSystemPropertyProviderRule.TestEnvironmentFacade() From e0f72cb6f01e2a85cb00beb359e8469efb442fda Mon Sep 17 00:00:00 2001 From: tbradellis Date: Wed, 9 Mar 2022 10:33:47 -0800 Subject: [PATCH 78/96] local decorating config tests added fix system prop name --- ...ationLoggingLocalDecoratingConfigTest.java | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java index e05061314e..eb0b0cf77f 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java @@ -1,7 +1,66 @@ package com.newrelic.agent.config; -import static org.junit.Assert.*; +import com.newrelic.agent.SaveSystemPropertyProviderRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class ApplicationLoggingLocalDecoratingConfigTest { - // TODO + + private Map localProps; + + @Rule + public SaveSystemPropertyProviderRule saveSystemPropertyProviderRule = new SaveSystemPropertyProviderRule(); + + @Before + public void setup() { + localProps = new HashMap<>(); + } + + @Test + public void defaultLocalDecoratingConfig() { + ApplicationLoggingLocalDecoratingConfig config = new ApplicationLoggingLocalDecoratingConfig(localProps, + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertFalse(config.getEnabled()); + + } + + @Test + public void usesEnvVarForLocalDecoratingConfig() { + + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade( + Collections.singletonMap("NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", "true")) + )); + + ApplicationLoggingLocalDecoratingConfig config = new ApplicationLoggingLocalDecoratingConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertTrue(config.getEnabled()); + + } + + @Test + public void usesSysPropForLocalDecoratingConfig() { + Properties properties = new Properties(); + + properties.put("newrelic.config.application_logging.local_decorating.enabled", "" + "true"); + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(properties), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade() + )); + + ApplicationLoggingLocalDecoratingConfig config = new ApplicationLoggingLocalDecoratingConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT); + assertTrue(config.getEnabled()); + + } } \ No newline at end of file From 5c3bdc07cc5f25aa5add66dc468a93f9958802be Mon Sep 17 00:00:00 2001 From: tbradellis Date: Thu, 10 Mar 2022 13:19:04 -0800 Subject: [PATCH 79/96] fix typo string concat --- .../config/ApplicationLoggingLocalDecoratingConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java index eb0b0cf77f..5e9c1729dc 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingLocalDecoratingConfigTest.java @@ -52,7 +52,7 @@ public void usesEnvVarForLocalDecoratingConfig() { public void usesSysPropForLocalDecoratingConfig() { Properties properties = new Properties(); - properties.put("newrelic.config.application_logging.local_decorating.enabled", "" + "true"); + properties.put("newrelic.config.application_logging.local_decorating.enabled", "true"); SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( new SaveSystemPropertyProviderRule.TestSystemProps(properties), new SaveSystemPropertyProviderRule.TestEnvironmentFacade() From 608a4de2a3ccb231373bb8bde03215ffd81c6649 Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Wed, 9 Mar 2022 12:13:24 -0500 Subject: [PATCH 80/96] Implement LogSenderServiceImplTest --- .../service/logging/LogSenderServiceImpl.java | 2 +- .../logging/LogSenderServiceImplTest.java | 234 +++++++++++++++++- 2 files changed, 233 insertions(+), 3 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 462417077e..e70da55b53 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -184,7 +184,7 @@ private void removeHarvestables() { */ @Override public void recordLogEvent(Map attributes) { - if (logEventsDisabled()) { + if (logEventsDisabled() || attributes == null || attributes.isEmpty()) { return; } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java index 02dfd86803..430c79417e 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java @@ -1,7 +1,237 @@ package com.newrelic.agent.service.logging; -import static org.junit.Assert.*; +import com.google.common.collect.ImmutableMap; +import com.newrelic.agent.HarvestService; +import com.newrelic.agent.MockRPMService; +import com.newrelic.agent.RPMService; +import com.newrelic.agent.RPMServiceManager; +import com.newrelic.agent.Transaction; +import com.newrelic.agent.TransactionData; +import com.newrelic.agent.TransactionService; +import com.newrelic.agent.config.AgentConfigImpl; +import com.newrelic.agent.config.ApplicationLoggingConfigImpl; +import com.newrelic.agent.config.ApplicationLoggingForwardingConfig; +import com.newrelic.agent.config.ApplicationLoggingLocalDecoratingConfig; +import com.newrelic.agent.config.ApplicationLoggingMetricsConfig; +import com.newrelic.agent.config.ConfigService; +import com.newrelic.agent.service.ServiceFactory; +import com.newrelic.agent.service.ServiceManager; +import com.newrelic.agent.stats.StatsService; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class LogSenderServiceImplTest { - // TODO + + private static final String appName = LogSenderServiceImplTest.class.getSimpleName() + "App"; + private static final HarvestService harvestService = Mockito.mock(HarvestService.class); + private static final TransactionService txService = Mockito.mock(TransactionService.class); + private static final StatsService statsService = Mockito.mock(StatsService.class); + + private static LogSenderServiceImpl createService() throws Exception { + return createService(createConfig()); + } + + private static LogSenderServiceImpl createService(Map config) throws Exception { + config = new HashMap<>(config); + + ServiceManager serviceManager = mock(ServiceManager.class); + when(serviceManager.getHarvestService()).thenReturn(harvestService); + when(serviceManager.getStatsService()).thenReturn(statsService); + when(serviceManager.getTransactionService()).thenReturn(txService); + when(serviceManager.getRPMServiceManager()).thenReturn(Mockito.mock(RPMServiceManager.class)); + when(serviceManager.getRPMServiceManager().getRPMService()).thenReturn(Mockito.mock(RPMService.class)); + when(serviceManager.getConfigService()).thenReturn(Mockito.mock(ConfigService.class)); + when(serviceManager.getConfigService().getDefaultAgentConfig()).thenReturn(AgentConfigImpl.createAgentConfig(config)); + when(serviceManager.getRPMServiceManager().getRPMService().getApplicationName()).thenReturn(appName); + ServiceFactory.setServiceManager(serviceManager); + + Transaction transaction = Mockito.mock(Transaction.class); + when(txService.getTransaction(false)).thenReturn(transaction); + + LogSenderServiceImpl logSenderService = new LogSenderServiceImpl(); + when(ServiceFactory.getServiceManager().getLogSenderService()).thenReturn(logSenderService); + + logSenderService.start(); + + return logSenderService; + } + + @Test + public void testHarvestableConfigure() throws Exception { + Map config = createConfig(true, 180); + LogSenderServiceImpl logSenderService = createService(config); + logSenderService.addHarvestableToService(appName); + logSenderService.configureHarvestables(60, 1); + + assertEquals(1, logSenderService.getMaxSamplesStored()); + } + + @Test + public void testHighSecurity() throws Exception { + Map config = createConfig(true, 180); + LogSenderServiceImpl logSenderService = createService(config); + + Transaction transaction = Mockito.mock(Transaction.class); + when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); + + LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( + AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + when(transaction.getLogEventData()).thenReturn(logs); + when(transaction.getApplicationName()).thenReturn(appName); + when(transaction.isInProgress()).thenReturn(true); + + logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); + logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); + logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + + MockRPMService analyticsData = new MockRPMService(); + when(ServiceFactory.getServiceManager().getRPMServiceManager().getRPMService(appName)).thenReturn( + analyticsData); + + TransactionData transactionData = Mockito.mock(TransactionData.class); + when(transactionData.getApplicationName()).thenReturn(appName); + when(transactionData.getLogEventData()).thenReturn(logs); + + logSenderService.transactionListener.dispatcherTransactionFinished(transactionData, null); + logSenderService.harvestHarvestables(); + + assertEquals(0, analyticsData.getEvents().size()); + assertEquals(0, logs.events.size()); + } + + @Test + public void testNoTransaction() throws Exception { + LogSenderServiceImpl logSenderService = createService(); + + logSenderService.addHarvestableToService(appName); + + verify(txService, times(1)).addTransactionListener(logSenderService.transactionListener); + + logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); + logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); + logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + + MockRPMService analyticsData = new MockRPMService(); + when(ServiceFactory.getServiceManager().getRPMServiceManager().getOrCreateRPMService(appName)).thenReturn( + analyticsData); + + logSenderService.harvestHarvestables(); + + assertEquals(3, analyticsData.getEvents().size()); + + logSenderService.stop(); + + verify(txService, times(1)).removeTransactionListener(logSenderService.transactionListener); + } + + @Test + public void testWithTransaction() throws Exception { + LogSenderServiceImpl logSenderService = createService(createConfig(null, 180)); + Transaction transaction = Mockito.mock(Transaction.class); + when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); + + LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( + AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + when(transaction.getLogEventData()).thenReturn(logs); + when(transaction.getApplicationName()).thenReturn(appName); + when(transaction.isInProgress()).thenReturn(true); + + logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); + logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); + logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + + MockRPMService analyticsData = new MockRPMService(); + when(ServiceFactory.getServiceManager().getRPMServiceManager().getOrCreateRPMService(appName)).thenReturn( + analyticsData); + + logSenderService.harvestHarvestables(); + + assertEquals(0, analyticsData.getEvents().size()); + assertEquals(3, logs.events.size()); + } + + @Test + public void testTransactionHarvest() throws Exception { + LogSenderServiceImpl logSenderService = createService(createConfig(null, 180)); + logSenderService.addHarvestableToService(appName); + + Transaction transaction = Mockito.mock(Transaction.class); + when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); + + LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( + AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + when(transaction.getLogEventData()).thenReturn(logs); + when(transaction.getApplicationName()).thenReturn(appName); + when(transaction.isInProgress()).thenReturn(true); + + logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); + logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); + logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + + // these should be filtered out + logSenderService.recordLogEvent(null); + logSenderService.recordLogEvent(Collections.emptyMap()); + + MockRPMService analyticsData = new MockRPMService(); + when(ServiceFactory.getServiceManager().getRPMServiceManager().getOrCreateRPMService(appName)).thenReturn( + analyticsData); + + TransactionData transactionData = Mockito.mock(TransactionData.class); + when(transactionData.getApplicationName()).thenReturn(appName); + when(transactionData.getLogEventData()).thenReturn(logs); + + logSenderService.transactionListener.dispatcherTransactionFinished(transactionData, null); + logSenderService.harvestHarvestables(); + + logSenderService.harvestHarvestables(); + + assertEquals(3, analyticsData.getEvents().size()); + } + + + private static Map createConfig() { + return createConfig(null, null, null); + } + + private static Map createConfig(Boolean highSecurity, Integer asyncTimeout) { + return createConfig(highSecurity, asyncTimeout, null); + } + + private static Map createConfig(Boolean highSecurity, Integer asyncTimeout, Long maxSamplesStored) { + Map subForwardingMap = new HashMap<>(); + subForwardingMap.put(ApplicationLoggingForwardingConfig.ENABLED, true); + subForwardingMap.put(ApplicationLoggingForwardingConfig.MAX_SAMPLES_STORED, maxSamplesStored); + + Map subMetricMap = new HashMap<>(); + subMetricMap.put(ApplicationLoggingMetricsConfig.ENABLED, true); + + Map subDecoratingMap = new HashMap<>(); + subDecoratingMap.put(ApplicationLoggingLocalDecoratingConfig.ENABLED, true); + + Map loggingMap = new HashMap<>(); + loggingMap.put(ApplicationLoggingConfigImpl.FORWARDING, subForwardingMap); + loggingMap.put(ApplicationLoggingConfigImpl.METRICS, subMetricMap); + loggingMap.put(ApplicationLoggingConfigImpl.LOCAL_DECORATING, subDecoratingMap); + + Map config = new HashMap<>(); + config.put(AgentConfigImpl.APPLICATION_LOGGING, loggingMap); + if (highSecurity != null) { + config.put(AgentConfigImpl.HIGH_SECURITY, highSecurity); + } + if (asyncTimeout != null) { + config.put(AgentConfigImpl.ASYNC_TIMEOUT, asyncTimeout); + } + config.put(AgentConfigImpl.APP_NAME, appName); + return config; + } } \ No newline at end of file From fc828c3039776e60e89ea24adf9dc933f9271a4c Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 14 Mar 2022 17:38:52 -0700 Subject: [PATCH 81/96] Move adding of agent metadata to LogSenderServiceImpl instead of individual instrumentation modules --- .../instrumentation/log4j2/AgentUtil.java | 34 +---------- .../logbackclassic12/AgentUtil.java | 32 +--------- .../java/com/newrelic/agent/AgentImpl.java | 60 ++++++++++++++++--- .../service/logging/LogSenderServiceImpl.java | 5 +- 4 files changed, 61 insertions(+), 70 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index a58156de32..684323f932 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -13,20 +13,16 @@ import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.message.Message; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Set; public class AgentUtil { + public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; // Log message attributes public static final String MESSAGE = "message"; public static final String TIMESTAMP = "timestamp"; public static final String LOG_LEVEL = "log.level"; public static final String UNKNOWN = "UNKNOWN"; - // Linking metadata attributes to filter out - private static final String ENTITY_TYPE = "entity.type"; - private static final String ENTITY_NAME = "entity.name"; // Linking metadata attributes used in blob private static final String BLOB_PREFIX = "NR-LINKING"; private static final String BLOB_DELIMITER = "|"; @@ -52,7 +48,7 @@ public static void recordNewRelicLogEvent(LogEvent event) { String formattedMessage = message.getFormattedMessage(); // Bail out and don't create a LogEvent if log message is empty if (formattedMessage != null && !formattedMessage.isEmpty()) { - HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); + HashMap logEventMap = new HashMap<>(DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES); logEventMap.put(MESSAGE, formattedMessage); logEventMap.put(TIMESTAMP, event.getTimeMillis()); @@ -100,32 +96,6 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } - /** - * Gets a map of agent linking metadata after filtering out - * entity.type, entity.name, and any attributes with an empty value. - * - * @return Filtered map of agent linking metadata - */ - public static Map getFilteredLinkingMetadataMap() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - Map map = new HashMap<>(); - Set> metadataSet = agentLinkingMetadata.entrySet(); - - for (Map.Entry entry : metadataSet) { - String key = entry.getKey(); - String value = entry.getValue(); - if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { - map.put(key, value); - } - } - return map; - } else { - return Collections.emptyMap(); - } - } - /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index c1e9f635fb..7661943dc3 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -17,14 +17,12 @@ import java.util.Set; public class AgentUtil { + public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; // Log message attributes public static final String MESSAGE = "message"; public static final String TIMESTAMP = "timestamp"; public static final String LOG_LEVEL = "log.level"; public static final String UNKNOWN = "UNKNOWN"; - // Linking metadata attributes to filter out - private static final String ENTITY_TYPE = "entity.type"; - private static final String ENTITY_NAME = "entity.name"; // Linking metadata attributes used in blob private static final String BLOB_PREFIX = "NR-LINKING"; private static final String BLOB_DELIMITER = "|"; @@ -48,7 +46,7 @@ public class AgentUtil { public static void recordNewRelicLogEvent(String message, long timeStampMillis, Level level) { // Bail out and don't create a LogEvent if log message is empty if (!message.isEmpty()) { - HashMap logEventMap = new HashMap<>(getFilteredLinkingMetadataMap()); + HashMap logEventMap = new HashMap<>(DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES); logEventMap.put(MESSAGE, message); logEventMap.put(TIMESTAMP, timeStampMillis); @@ -89,32 +87,6 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } - /** - * Gets a map of agent linking metadata after filtering out - * entity.type, entity.name, and any attributes with an empty value. - * - * @return Filtered map of agent linking metadata - */ - public static Map getFilteredLinkingMetadataMap() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - Map map = new HashMap<>(); - Set> metadataSet = agentLinkingMetadata.entrySet(); - - for (Map.Entry entry : metadataSet) { - String key = entry.getKey(); - String value = entry.getValue(); - if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { - map.put(key, value); - } - } - return map; - } else { - return Collections.emptyMap(); - } - } - /** * Check if all application_logging features are enabled. * diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java index a804458839..646f97fbfb 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java @@ -30,6 +30,13 @@ public class AgentImpl implements com.newrelic.agent.bridge.Agent { private final Logger logger; + private static final String TRACE_ID = "trace.id"; + private static final String SPAN_ID = "span.id"; + private static final String HOSTNAME = "hostname"; + private static final String ENTITY_GUID = "entity.guid"; + private static final String ENTITY_NAME = "entity.name"; + private static final String ENTITY_TYPE = "entity.type"; + public AgentImpl(Logger logger) { this.logger = logger; } @@ -157,19 +164,58 @@ public Map getLinkingMetadata() { TraceMetadata traceMetadata = getTraceMetadata(); String traceId = traceMetadata.getTraceId(); - linkingMetadata.put("trace.id", traceId); + linkingMetadata.put(TRACE_ID, traceId); String spanId = traceMetadata.getSpanId(); - linkingMetadata.put("span.id", spanId); + linkingMetadata.put(SPAN_ID, spanId); AgentConfig agentConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); - linkingMetadata.put("hostname", getLinkingMetaHostname(agentConfig)); + linkingMetadata.put(HOSTNAME, getLinkingMetaHostname(agentConfig)); + try { + String entityGuid = ServiceFactory.getRPMService().getEntityGuid(); + if (!entityGuid.isEmpty()) { + linkingMetadata.put(ENTITY_NAME, agentConfig.getApplicationName()); + linkingMetadata.put(ENTITY_TYPE, "SERVICE"); + linkingMetadata.put(ENTITY_GUID, entityGuid); + } + } catch (NullPointerException ignored) { + // it's possible to call getLinkingMetadata in the premain before RPMService has been initialized which will NPE + Agent.LOG.log(Level.WARNING, "Cannot get entity.guid from getLinkingMetadata() until RPMService has initialized."); + } + + return linkingMetadata; + } + + /** + * Gets a map of agent linking metadata minus entity.type, + * entity.name, and any attributes with an empty value. + * This subset of linking metadata is added to LogEvents. + * + * @return Filtered map of agent linking metadata + */ + public static Map getLogEventLinkingMetadata() { + Map linkingMetadata = new ConcurrentHashMap<>(); + + TraceMetadata traceMetadata = TraceMetadataImpl.INSTANCE; + String traceId = traceMetadata.getTraceId(); + if (!traceId.isEmpty()) { + linkingMetadata.put(TRACE_ID, traceId); + } + + String spanId = traceMetadata.getSpanId(); + if (!spanId.isEmpty()) { + linkingMetadata.put(SPAN_ID, spanId); + } + + AgentConfig agentConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); + String hostname = getLinkingMetaHostname(agentConfig); + if (!hostname.isEmpty()) { + linkingMetadata.put(HOSTNAME, hostname); + } try { String entityGuid = ServiceFactory.getRPMService().getEntityGuid(); if (!entityGuid.isEmpty()) { - linkingMetadata.put("entity.name", agentConfig.getApplicationName()); - linkingMetadata.put("entity.type", "SERVICE"); - linkingMetadata.put("entity.guid", entityGuid); + linkingMetadata.put(ENTITY_GUID, entityGuid); } } catch (NullPointerException ignored) { // it's possible to call getLinkingMetadata in the premain before RPMService has been initialized which will NPE @@ -179,7 +225,7 @@ public Map getLinkingMetadata() { return linkingMetadata; } - private String getLinkingMetaHostname(AgentConfig agentConfig) { + private static String getLinkingMetaHostname(AgentConfig agentConfig) { String fullHostname = Hostname.getFullHostname(agentConfig); if (fullHostname == null || fullHostname.isEmpty() || fullHostname.equals("localhost")) { return Hostname.getHostname(agentConfig); diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 462417077e..b0a93322b9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -11,6 +11,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import com.newrelic.agent.Agent; +import com.newrelic.agent.AgentImpl; import com.newrelic.agent.ExtendedTransactionListener; import com.newrelic.agent.Harvestable; import com.newrelic.agent.MetricNames; @@ -471,7 +472,9 @@ private static String mapInternString(String value) { * @return LogEvent instance */ private static LogEvent createValidatedEvent(Map attributes) { - Map userAttributes = new HashMap<>(attributes.size()); + // Initialize new userAttributes map with agent linking metadata required for LogEvents + Map userAttributes = new HashMap<>(AgentImpl.getLogEventLinkingMetadata()); + LogEvent event = new LogEvent(userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); // Now add the attributes from the argument map to the event using an AttributeSender. From 0ff81af3be5c0fcf8432a3ac4faafb437d9dddf2 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Mar 2022 16:20:24 -0700 Subject: [PATCH 82/96] Refactor for better testability --- .../java/com/newrelic/agent/AgentImpl.java | 83 +---------- .../newrelic/agent/AgentLinkingMetadata.java | 137 ++++++++++++++++++ .../service/logging/LogSenderServiceImpl.java | 21 +-- 3 files changed, 151 insertions(+), 90 deletions(-) create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java index 646f97fbfb..c3e73e7a0e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentImpl.java @@ -12,8 +12,6 @@ import com.newrelic.agent.bridge.NoOpTransaction; import com.newrelic.agent.bridge.TracedMethod; import com.newrelic.agent.bridge.Transaction; -import com.newrelic.agent.config.AgentConfig; -import com.newrelic.agent.config.Hostname; import com.newrelic.agent.service.ServiceFactory; import com.newrelic.agent.tracers.Tracer; import com.newrelic.api.agent.Insights; @@ -23,20 +21,12 @@ import com.newrelic.api.agent.TraceMetadata; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; public class AgentImpl implements com.newrelic.agent.bridge.Agent { private final Logger logger; - private static final String TRACE_ID = "trace.id"; - private static final String SPAN_ID = "span.id"; - private static final String HOSTNAME = "hostname"; - private static final String ENTITY_GUID = "entity.guid"; - private static final String ENTITY_NAME = "entity.name"; - private static final String ENTITY_TYPE = "entity.type"; - public AgentImpl(Logger logger) { this.logger = logger; } @@ -159,78 +149,9 @@ public TraceMetadata getTraceMetadata() { return TraceMetadataImpl.INSTANCE; } + @Override public Map getLinkingMetadata() { - Map linkingMetadata = new ConcurrentHashMap<>(); - - TraceMetadata traceMetadata = getTraceMetadata(); - String traceId = traceMetadata.getTraceId(); - linkingMetadata.put(TRACE_ID, traceId); - - String spanId = traceMetadata.getSpanId(); - linkingMetadata.put(SPAN_ID, spanId); - - AgentConfig agentConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); - linkingMetadata.put(HOSTNAME, getLinkingMetaHostname(agentConfig)); - try { - String entityGuid = ServiceFactory.getRPMService().getEntityGuid(); - if (!entityGuid.isEmpty()) { - linkingMetadata.put(ENTITY_NAME, agentConfig.getApplicationName()); - linkingMetadata.put(ENTITY_TYPE, "SERVICE"); - linkingMetadata.put(ENTITY_GUID, entityGuid); - } - } catch (NullPointerException ignored) { - // it's possible to call getLinkingMetadata in the premain before RPMService has been initialized which will NPE - Agent.LOG.log(Level.WARNING, "Cannot get entity.guid from getLinkingMetadata() until RPMService has initialized."); - } - - return linkingMetadata; - } - - /** - * Gets a map of agent linking metadata minus entity.type, - * entity.name, and any attributes with an empty value. - * This subset of linking metadata is added to LogEvents. - * - * @return Filtered map of agent linking metadata - */ - public static Map getLogEventLinkingMetadata() { - Map linkingMetadata = new ConcurrentHashMap<>(); - - TraceMetadata traceMetadata = TraceMetadataImpl.INSTANCE; - String traceId = traceMetadata.getTraceId(); - if (!traceId.isEmpty()) { - linkingMetadata.put(TRACE_ID, traceId); - } - - String spanId = traceMetadata.getSpanId(); - if (!spanId.isEmpty()) { - linkingMetadata.put(SPAN_ID, spanId); - } - - AgentConfig agentConfig = ServiceFactory.getConfigService().getDefaultAgentConfig(); - String hostname = getLinkingMetaHostname(agentConfig); - if (!hostname.isEmpty()) { - linkingMetadata.put(HOSTNAME, hostname); - } - try { - String entityGuid = ServiceFactory.getRPMService().getEntityGuid(); - if (!entityGuid.isEmpty()) { - linkingMetadata.put(ENTITY_GUID, entityGuid); - } - } catch (NullPointerException ignored) { - // it's possible to call getLinkingMetadata in the premain before RPMService has been initialized which will NPE - Agent.LOG.log(Level.WARNING, "Cannot get entity.guid from getLinkingMetadata() until RPMService has initialized."); - } - - return linkingMetadata; - } - - private static String getLinkingMetaHostname(AgentConfig agentConfig) { - String fullHostname = Hostname.getFullHostname(agentConfig); - if (fullHostname == null || fullHostname.isEmpty() || fullHostname.equals("localhost")) { - return Hostname.getHostname(agentConfig); - } - return fullHostname; + return AgentLinkingMetadata.getLinkingMetadata(getTraceMetadata(), ServiceFactory.getConfigService(), ServiceFactory.getRPMService()); } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java new file mode 100644 index 0000000000..c3c994d44a --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java @@ -0,0 +1,137 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent; + +import com.newrelic.agent.config.AgentConfig; +import com.newrelic.agent.config.ConfigService; +import com.newrelic.agent.config.Hostname; +import com.newrelic.api.agent.TraceMetadata; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; + +/** + * Utility class for providing agent linking metadata. + * This metadata can be used to link events to specific traces, spans, hosts, and entities. + */ +public class AgentLinkingMetadata { + public static final String ENTITY_TYPE_DEFAULT = "SERVICE"; + public static final String LOCALHOST = "localhost"; + // Agent linking metadata attributes + public static final String TRACE_ID = "trace.id"; + public static final String SPAN_ID = "span.id"; + public static final String HOSTNAME = "hostname"; + public static final String ENTITY_GUID = "entity.guid"; + public static final String ENTITY_NAME = "entity.name"; + public static final String ENTITY_TYPE = "entity.type"; + + /** + * Get a map of all agent linking metadata. + * + * @param traceMetadata TraceMetadataImpl instance to get spanId and traceId + * @param configService ConfigService to get hostName and entityName + * @param rpmService IRPMService to get entityGuid + * @return Map of all agent linking metadata + */ + public static Map getLinkingMetadata(TraceMetadata traceMetadata, ConfigService configService, IRPMService rpmService) { + AgentConfig agentConfig = configService.getDefaultAgentConfig(); + Map linkingMetadata = new ConcurrentHashMap<>(); + + linkingMetadata.put(TRACE_ID, getTraceId(traceMetadata)); + linkingMetadata.put(SPAN_ID, getSpanId(traceMetadata)); + linkingMetadata.put(HOSTNAME, getHostname(agentConfig)); + + try { + String entityGuid = getEntityGuid(rpmService); + if (!entityGuid.isEmpty()) { + linkingMetadata.put(ENTITY_NAME, getEntityName(agentConfig)); + linkingMetadata.put(ENTITY_TYPE, getEntityType()); + linkingMetadata.put(ENTITY_GUID, entityGuid); + } + } catch (NullPointerException ignored) { + logWarning(); + } + + return linkingMetadata; + } + + /** + * Get a map of agent linking metadata minus entity.type, + * entity.name, and any attributes with an empty value. + * This subset of linking metadata is added to LogEvents. + * + * @param traceMetadata TraceMetadataImpl to get spanId and traceId + * @param configService ConfigService to get hostName and entityName + * @param rpmService IRPMService to get entityGuid + * @return Filtered map of agent linking metadata + */ + public static Map getLogEventLinkingMetadata(TraceMetadata traceMetadata, ConfigService configService, IRPMService rpmService) { + AgentConfig agentConfig = configService.getDefaultAgentConfig(); + Map logEventLinkingMetadata = new ConcurrentHashMap<>(); + + String traceId = getTraceId(traceMetadata); + if (!traceId.isEmpty()) { + logEventLinkingMetadata.put(TRACE_ID, traceId); + } + + String spanId = getSpanId(traceMetadata); + if (!spanId.isEmpty()) { + logEventLinkingMetadata.put(SPAN_ID, spanId); + } + + String hostname = getHostname(agentConfig); + if (!hostname.isEmpty()) { + logEventLinkingMetadata.put(HOSTNAME, hostname); + } + try { + String entityGuid = rpmService.getEntityGuid(); + if (!entityGuid.isEmpty()) { + logEventLinkingMetadata.put(ENTITY_GUID, entityGuid); + } + } catch (NullPointerException ignored) { + logWarning(); + } + + return logEventLinkingMetadata; + } + + public static String getTraceId(TraceMetadata traceMetadata) { + return traceMetadata.getTraceId(); + } + + public static String getSpanId(TraceMetadata traceMetadata) { + return traceMetadata.getSpanId(); + } + + private static String getHostname(AgentConfig agentConfig) { + String fullHostname = Hostname.getFullHostname(agentConfig); + if (fullHostname == null || fullHostname.isEmpty() || fullHostname.equals(LOCALHOST)) { + return Hostname.getHostname(agentConfig); + } + return fullHostname; + } + + public static String getEntityName(AgentConfig agentConfig) { + return agentConfig.getApplicationName(); + } + + public static String getEntityType() { + return ENTITY_TYPE_DEFAULT; + } + + public static String getEntityGuid(IRPMService rpmService) { + return rpmService.getEntityGuid(); + } + + private static void logWarning() { + // It's possible to call getEntityGuid in the agent premain before the + // RPMService has been initialized, which will cause a NullPointerException. + Agent.LOG.log(Level.WARNING, "Cannot get entity.guid from getLinkingMetadata() until RPMService has initialized."); + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index b0a93322b9..ef921e9bf0 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -11,10 +11,11 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import com.newrelic.agent.Agent; -import com.newrelic.agent.AgentImpl; +import com.newrelic.agent.AgentLinkingMetadata; import com.newrelic.agent.ExtendedTransactionListener; import com.newrelic.agent.Harvestable; import com.newrelic.agent.MetricNames; +import com.newrelic.agent.TraceMetadataImpl; import com.newrelic.agent.Transaction; import com.newrelic.agent.TransactionData; import com.newrelic.agent.attributes.AttributeSender; @@ -472,17 +473,19 @@ private static String mapInternString(String value) { * @return LogEvent instance */ private static LogEvent createValidatedEvent(Map attributes) { - // Initialize new userAttributes map with agent linking metadata required for LogEvents - Map userAttributes = new HashMap<>(AgentImpl.getLogEventLinkingMetadata()); + Map logEventLinkingMetadata = AgentLinkingMetadata.getLogEventLinkingMetadata(TraceMetadataImpl.INSTANCE, + ServiceFactory.getConfigService(), ServiceFactory.getRPMService()); + // Initialize new logEventAttributes map with agent linking metadata + Map logEventAttributes = new HashMap<>(logEventLinkingMetadata); - LogEvent event = new LogEvent(userAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); + LogEvent event = new LogEvent(logEventAttributes, DistributedTraceServiceImpl.nextTruncatedFloat()); // Now add the attributes from the argument map to the event using an AttributeSender. // An AttributeSender is the way to reuse all the existing attribute validations. We // also locally "intern" Strings because we anticipate a lot of reuse of the keys and, // possibly, the values. But there's an interaction: if the key or value is chopped // within the attribute sender, the modified value won't be "interned" in our map. - AttributeSender sender = new LogEventAttributeSender(userAttributes); + AttributeSender sender = new LogEventAttributeSender(logEventAttributes); for (Map.Entry entry : attributes.entrySet()) { String key = entry.getKey(); @@ -519,11 +522,11 @@ private static class LogEventAttributeSender extends AttributeSender { private static final String ATTRIBUTE_TYPE = "log"; - private final Map userAttributes; + private final Map logEventAttributes; - public LogEventAttributeSender(Map userAttributes) { + public LogEventAttributeSender(Map logEventAttributes) { super(new AttributeValidator(ATTRIBUTE_TYPE)); - this.userAttributes = userAttributes; + this.logEventAttributes = logEventAttributes; setTransactional(false); } @@ -535,7 +538,7 @@ protected String getAttributeType() { @Override protected Map getAttributeMap() { if (ServiceFactory.getConfigService().getDefaultAgentConfig().isCustomParametersAllowed()) { - return userAttributes; + return logEventAttributes; } return null; } From d2441212633c99d0e18a94b0b3c72a5a10071896 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Mar 2022 16:21:45 -0700 Subject: [PATCH 83/96] Remove linking metadata map filtering from other instrumentation modules --- .../instrumentation/log4j1/AgentUtil.java | 32 ------------------- .../com/nr/instrumentation/jul/AgentUtil.java | 32 ------------------- 2 files changed, 64 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 0b5e068d9d..77a8d5dd6d 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -9,10 +9,7 @@ import com.newrelic.api.agent.NewRelic; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.Set; public class AgentUtil { // Log message attributes @@ -20,9 +17,6 @@ public class AgentUtil { public static final String TIMESTAMP = "timestamp"; public static final String LOG_LEVEL = "log.level"; public static final String UNKNOWN = "UNKNOWN"; - // Linking metadata attributes to filter out - private static final String ENTITY_TYPE = "entity.type"; - private static final String ENTITY_NAME = "entity.name"; // Linking metadata attributes used in blob private static final String BLOB_PREFIX = "NR-LINKING"; private static final String BLOB_DELIMITER = "|"; @@ -63,32 +57,6 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } - /** - * Gets a map of agent linking metadata after filtering out - * entity.type, entity.name, and any attributes with an empty value. - * - * @return Filtered map of agent linking metadata - */ - public static Map getFilteredLinkingMetadataMap() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - Map map = new HashMap<>(); - Set> metadataSet = agentLinkingMetadata.entrySet(); - - for (Map.Entry entry : metadataSet) { - String key = entry.getKey(); - String value = entry.getValue(); - if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { - map.put(key, value); - } - } - return map; - } else { - return Collections.emptyMap(); - } - } - /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index e18c2d6b19..37cb2f56aa 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -9,10 +9,7 @@ import com.newrelic.api.agent.NewRelic; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.Set; public class AgentUtil { // Log message attributes @@ -20,9 +17,6 @@ public class AgentUtil { public static final String TIMESTAMP = "timestamp"; public static final String LOG_LEVEL = "log.level"; public static final String UNKNOWN = "UNKNOWN"; - // Linking metadata attributes to filter out - private static final String ENTITY_TYPE = "entity.type"; - private static final String ENTITY_NAME = "entity.name"; // Linking metadata attributes used in blob private static final String BLOB_PREFIX = "NR-LINKING"; private static final String BLOB_DELIMITER = "|"; @@ -63,32 +57,6 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } - /** - * Gets a map of agent linking metadata after filtering out - * entity.type, entity.name, and any attributes with an empty value. - * - * @return Filtered map of agent linking metadata - */ - public static Map getFilteredLinkingMetadataMap() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - Map map = new HashMap<>(); - Set> metadataSet = agentLinkingMetadata.entrySet(); - - for (Map.Entry entry : metadataSet) { - String key = entry.getKey(); - String value = entry.getValue(); - if (!key.equals(ENTITY_NAME) && !key.equals(ENTITY_TYPE) && !value.isEmpty()) { - map.put(key, value); - } - } - return map; - } else { - return Collections.emptyMap(); - } - } - /** * Check if all application_logging features are enabled. * From 2e87c55a7ae64c3a648b57bc2f271f97cd236af2 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 16 Mar 2022 16:25:08 -0700 Subject: [PATCH 84/96] Add constant to other instrumentation modules --- .../main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java | 1 + .../src/main/java/com/nr/instrumentation/jul/AgentUtil.java | 1 + 2 files changed, 2 insertions(+) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 77a8d5dd6d..2a8f4ee2d3 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -12,6 +12,7 @@ import java.util.Map; public class AgentUtil { + public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; // Log message attributes public static final String MESSAGE = "message"; public static final String TIMESTAMP = "timestamp"; diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 37cb2f56aa..3db291bb37 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -12,6 +12,7 @@ import java.util.Map; public class AgentUtil { + public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; // Log message attributes public static final String MESSAGE = "message"; public static final String TIMESTAMP = "timestamp"; From 1d3c70fa079c5a543c3abeb71b74874b9f279145 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Mar 2022 10:36:34 -0700 Subject: [PATCH 85/96] Add AgentLinkingMetadataTest --- .../agent/AgentLinkingMetadataTest.java | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java new file mode 100644 index 0000000000..81bcf6716a --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java @@ -0,0 +1,198 @@ +package com.newrelic.agent; + +import com.newrelic.agent.config.AgentConfigImpl; +import com.newrelic.agent.config.ConfigServiceImpl; +import com.newrelic.agent.service.ServiceFactory; +import com.newrelic.agent.service.ServiceManagerImpl; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AgentLinkingMetadataTest { + + @Test + public void getLinkingMetadata() { + // Given + final String expectedTraceId = "traceId1234"; + final String expectedSpanId = "spanId5678"; + final String expectedEntityGuid = "entityGuid91011"; + final String expectedEntityName = "entityName91011"; + final String expectedEntityType = AgentLinkingMetadata.ENTITY_TYPE_DEFAULT; + + TraceMetadataImpl traceMetadataMock = mock(TraceMetadataImpl.class); + ServiceManagerImpl serviceManagerMock = mock(ServiceManagerImpl.class); + RPMServiceManagerImpl rpmServiceManagerMock = mock(RPMServiceManagerImpl.class); + RPMService rpmServiceMock = mock(RPMService.class); + ConfigServiceImpl configServiceMock = mock(ConfigServiceImpl.class); + AgentConfigImpl agentConfigMock = mock(AgentConfigImpl.class); + + ServiceFactory.setServiceManager(serviceManagerMock); + + // When + when(traceMetadataMock.getTraceId()).thenReturn(expectedTraceId); + when(traceMetadataMock.getSpanId()).thenReturn(expectedSpanId); + when(serviceManagerMock.getRPMServiceManager()).thenReturn(rpmServiceManagerMock); + when(serviceManagerMock.getConfigService()).thenReturn(configServiceMock); + when(rpmServiceManagerMock.getRPMService()).thenReturn(rpmServiceMock); + when(configServiceMock.getDefaultAgentConfig()).thenReturn(agentConfigMock); + when(agentConfigMock.getApplicationName()).thenReturn(expectedEntityName); + when(rpmServiceMock.getEntityGuid()).thenReturn(expectedEntityGuid); + + // Then + Map linkingMetadata = AgentLinkingMetadata.getLinkingMetadata(traceMetadataMock, ServiceFactory.getConfigService(), + ServiceFactory.getRPMService()); + + assertFalse("linkingMetadata map shouldn't be empty", linkingMetadata.isEmpty()); + + // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test + assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); + + assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); + assertEquals(expectedEntityName, linkingMetadata.get(AgentLinkingMetadata.ENTITY_NAME)); + assertEquals(expectedEntityType, linkingMetadata.get(AgentLinkingMetadata.ENTITY_TYPE)); + + assertEquals(expectedTraceId, linkingMetadata.get(AgentLinkingMetadata.TRACE_ID)); + assertEquals(expectedSpanId, linkingMetadata.get(AgentLinkingMetadata.SPAN_ID)); + } + + @Test + public void getLinkingMetadataWithEmptyTraceAttributes() { + // Given + final String expectedTraceId = ""; + final String expectedSpanId = ""; + final String expectedEntityGuid = "entityGuid91011"; + final String expectedEntityName = "entityName91011"; + final String expectedEntityType = AgentLinkingMetadata.ENTITY_TYPE_DEFAULT; + + TraceMetadataImpl traceMetadataMock = mock(TraceMetadataImpl.class); + ServiceManagerImpl serviceManagerMock = mock(ServiceManagerImpl.class); + RPMServiceManagerImpl rpmServiceManagerMock = mock(RPMServiceManagerImpl.class); + RPMService rpmServiceMock = mock(RPMService.class); + ConfigServiceImpl configServiceMock = mock(ConfigServiceImpl.class); + AgentConfigImpl agentConfigMock = mock(AgentConfigImpl.class); + + ServiceFactory.setServiceManager(serviceManagerMock); + + // When + when(traceMetadataMock.getTraceId()).thenReturn(expectedTraceId); + when(traceMetadataMock.getSpanId()).thenReturn(expectedSpanId); + when(serviceManagerMock.getRPMServiceManager()).thenReturn(rpmServiceManagerMock); + when(serviceManagerMock.getConfigService()).thenReturn(configServiceMock); + when(rpmServiceManagerMock.getRPMService()).thenReturn(rpmServiceMock); + when(configServiceMock.getDefaultAgentConfig()).thenReturn(agentConfigMock); + when(agentConfigMock.getApplicationName()).thenReturn(expectedEntityName); + when(rpmServiceMock.getEntityGuid()).thenReturn(expectedEntityGuid); + + // Then + Map linkingMetadata = AgentLinkingMetadata.getLinkingMetadata(traceMetadataMock, ServiceFactory.getConfigService(), + ServiceFactory.getRPMService()); + + assertFalse("linkingMetadata map shouldn't be empty", linkingMetadata.isEmpty()); + + // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test + assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); + + assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); + assertEquals(expectedEntityName, linkingMetadata.get(AgentLinkingMetadata.ENTITY_NAME)); + assertEquals(expectedEntityType, linkingMetadata.get(AgentLinkingMetadata.ENTITY_TYPE)); + + // trace.id and span.id would be empty values if getLinkingMetadata was called outside of a transaction. + // With the getLinkingMetadata API the returned map includes keys with empty values + assertEquals(expectedTraceId, linkingMetadata.get(AgentLinkingMetadata.TRACE_ID)); + assertEquals(expectedSpanId, linkingMetadata.get(AgentLinkingMetadata.SPAN_ID)); + } + + @Test + public void getLogEventLinkingMetadata() { + // Given + final String expectedTraceId = "traceId1234"; + final String expectedSpanId = "spanId5678"; + final String expectedEntityGuid = "entityGuid91011"; + final String expectedEntityName = "entityName91011"; + + TraceMetadataImpl traceMetadataMock = mock(TraceMetadataImpl.class); + ServiceManagerImpl serviceManagerMock = mock(ServiceManagerImpl.class); + RPMServiceManagerImpl rpmServiceManagerMock = mock(RPMServiceManagerImpl.class); + RPMService rpmServiceMock = mock(RPMService.class); + ConfigServiceImpl configServiceMock = mock(ConfigServiceImpl.class); + AgentConfigImpl agentConfigMock = mock(AgentConfigImpl.class); + + ServiceFactory.setServiceManager(serviceManagerMock); + + // When + when(traceMetadataMock.getTraceId()).thenReturn(expectedTraceId); + when(traceMetadataMock.getSpanId()).thenReturn(expectedSpanId); + when(serviceManagerMock.getRPMServiceManager()).thenReturn(rpmServiceManagerMock); + when(serviceManagerMock.getConfigService()).thenReturn(configServiceMock); + when(rpmServiceManagerMock.getRPMService()).thenReturn(rpmServiceMock); + when(configServiceMock.getDefaultAgentConfig()).thenReturn(agentConfigMock); + when(agentConfigMock.getApplicationName()).thenReturn(expectedEntityName); + when(rpmServiceMock.getEntityGuid()).thenReturn(expectedEntityGuid); + + // Then + Map linkingMetadata = AgentLinkingMetadata.getLogEventLinkingMetadata(traceMetadataMock, ServiceFactory.getConfigService(), + ServiceFactory.getRPMService()); + + assertFalse("linkingMetadata map shouldn't be empty", linkingMetadata.isEmpty()); + + // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test + assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); + + assertFalse("entity.name should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_NAME)); + assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_TYPE)); + assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); + + assertEquals(expectedTraceId, linkingMetadata.get(AgentLinkingMetadata.TRACE_ID)); + assertEquals(expectedSpanId, linkingMetadata.get(AgentLinkingMetadata.SPAN_ID)); + } + + @Test + public void getLogEventLinkingMetadataWithEmptyTraceAttributes() { + // Given + final String expectedTraceId = ""; + final String expectedSpanId = ""; + final String expectedEntityGuid = "entityGuid91011"; + final String expectedEntityName = "entityName91011"; + + TraceMetadataImpl traceMetadataMock = mock(TraceMetadataImpl.class); + ServiceManagerImpl serviceManagerMock = mock(ServiceManagerImpl.class); + RPMServiceManagerImpl rpmServiceManagerMock = mock(RPMServiceManagerImpl.class); + RPMService rpmServiceMock = mock(RPMService.class); + ConfigServiceImpl configServiceMock = mock(ConfigServiceImpl.class); + AgentConfigImpl agentConfigMock = mock(AgentConfigImpl.class); + + ServiceFactory.setServiceManager(serviceManagerMock); + + // When + when(traceMetadataMock.getTraceId()).thenReturn(expectedTraceId); + when(traceMetadataMock.getSpanId()).thenReturn(expectedSpanId); + when(serviceManagerMock.getRPMServiceManager()).thenReturn(rpmServiceManagerMock); + when(serviceManagerMock.getConfigService()).thenReturn(configServiceMock); + when(rpmServiceManagerMock.getRPMService()).thenReturn(rpmServiceMock); + when(configServiceMock.getDefaultAgentConfig()).thenReturn(agentConfigMock); + when(agentConfigMock.getApplicationName()).thenReturn(expectedEntityName); + when(rpmServiceMock.getEntityGuid()).thenReturn(expectedEntityGuid); + + // Then + Map linkingMetadata = AgentLinkingMetadata.getLogEventLinkingMetadata(traceMetadataMock, ServiceFactory.getConfigService(), + ServiceFactory.getRPMService()); + + assertFalse("linkingMetadata map shouldn't be empty", linkingMetadata.isEmpty()); + + // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test + assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); + + assertFalse("entity.name should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_NAME)); + assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_TYPE)); + assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); + + // trace.id and span.id would be empty values if getLogEventLinkingMetadata was called outside of a transaction, in which case they are omitted + assertFalse("empty trace.id value should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.TRACE_ID)); + assertFalse("empty span.id value should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.SPAN_ID)); + } +} From e89c6cf7f107f018502c1c0a62ea7698d6006f1b Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Mar 2022 14:57:42 -0700 Subject: [PATCH 86/96] Log the number of events stored per harvest --- .../src/main/java/com/newrelic/agent/HarvestServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index d5395cc433..438f70cbc7 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -106,6 +106,8 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); if (harvestLimit != null) { + Agent.LOG.log(Level.FINE, "harvest limit from collector is: {0} samples stored per harvest for {1}", harvestLimit, + tracker.harvestable.getEndpointMethodName()); maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period ServiceFactory.getStatsService().doStatsWork( From fd36eae5ca045caa71275682b092e609205d0b26 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Mar 2022 15:08:41 -0700 Subject: [PATCH 87/96] Log the number of events stored per harvest for SpanEvents --- .../src/main/java/com/newrelic/agent/HarvestServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index 438f70cbc7..7e6914cc71 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -125,6 +125,8 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { tracker.harvestable.getEndpointMethodName()); Long harvestLimit = (Long) spanHarvestConfig.get(SERVER_SPAN_HARVEST_LIMIT); if (harvestLimit != null) { + Agent.LOG.log(Level.FINE, "harvest limit from collector is: {0} samples stored per harvest for {1}", harvestLimit, + tracker.harvestable.getEndpointMethodName()); maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) spanHarvestConfig.get(REPORT_PERIOD_MS); } From 78bfead6d43e46ed42b391beef1ebb5784a3ebaa Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Mar 2022 15:29:34 -0700 Subject: [PATCH 88/96] Clarify log message --- .../java/com/newrelic/agent/HarvestServiceImpl.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index 7e6914cc71..b2c9c39be9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -106,12 +106,13 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); if (harvestLimit != null) { - Agent.LOG.log(Level.FINE, "harvest limit from collector is: {0} samples stored per harvest for {1}", harvestLimit, - tracker.harvestable.getEndpointMethodName()); maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period + float reportPeriodInSeconds = reportPeriodInMillis / 1000; + Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} samples stored per every {2} second harvest", + tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); ServiceFactory.getStatsService().doStatsWork( - StatsWorks.getRecordMetricWork(MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS, reportPeriodInMillis / 1000), + StatsWorks.getRecordMetricWork(MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS, reportPeriodInSeconds), MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS ); } } else if (!isSpanEventEndpoint) { @@ -125,10 +126,11 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { tracker.harvestable.getEndpointMethodName()); Long harvestLimit = (Long) spanHarvestConfig.get(SERVER_SPAN_HARVEST_LIMIT); if (harvestLimit != null) { - Agent.LOG.log(Level.FINE, "harvest limit from collector is: {0} samples stored per harvest for {1}", harvestLimit, - tracker.harvestable.getEndpointMethodName()); maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) spanHarvestConfig.get(REPORT_PERIOD_MS); + float reportPeriodInSeconds = reportPeriodInMillis / 1000; + Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} samples stored per every {2} second harvest", + tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); } } else if (isSpanEventEndpoint) { Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector was null. Using default value: {0} samples stored for {1}", maxSamplesStored, From d943a50c05fd4c3d02cd8d9a0471d4adec5327d7 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Thu, 17 Mar 2022 15:49:04 -0700 Subject: [PATCH 89/96] Standardize log message format --- .../newrelic/agent/HarvestServiceImpl.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index b2c9c39be9..baa07a1a89 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -100,8 +100,8 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { // The event_harvest_config.harvest_limits received from server-side during the connect lifecycle // contains config for error_event_data, analytic_event_data, custom_event_data, and log_event_data if (eventHarvestConfig != null && !isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "event_harvest_config from collector is: {0} samples stored for {1}", maxSamplesStored, - tracker.harvestable.getEndpointMethodName()); + Agent.LOG.log(Level.FINE, "event_harvest_config from collector for {0} is: {1} max samples stored per minute", + tracker.harvestable.getEndpointMethodName(), maxSamplesStored); Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); @@ -109,32 +109,32 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period float reportPeriodInSeconds = reportPeriodInMillis / 1000; - Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} samples stored per every {2} second harvest", + Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} max samples stored per every {2} second harvest", tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); ServiceFactory.getStatsService().doStatsWork( StatsWorks.getRecordMetricWork(MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS, reportPeriodInSeconds), MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS ); } } else if (!isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "event_harvest_config from collector was null. Using default value: {0} samples stored for {1}", maxSamplesStored, - tracker.harvestable.getEndpointMethodName()); + Agent.LOG.log(Level.FINE, "event_harvest_config from collector for {0} was null. Using default value: {1} max samples stored per minute", + tracker.harvestable.getEndpointMethodName(), maxSamplesStored); } // The span_event_harvest_config received from server-side during the connect lifecycle contains config for span_event_data if (spanHarvestConfig != null && isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector is: {0} samples stored for {1}", maxSamplesStored, - tracker.harvestable.getEndpointMethodName()); + Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector for {0} is: {1} max samples stored per minute", + tracker.harvestable.getEndpointMethodName(), maxSamplesStored); Long harvestLimit = (Long) spanHarvestConfig.get(SERVER_SPAN_HARVEST_LIMIT); if (harvestLimit != null) { maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) spanHarvestConfig.get(REPORT_PERIOD_MS); float reportPeriodInSeconds = reportPeriodInMillis / 1000; - Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} samples stored per every {2} second harvest", + Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} max samples stored per every {2} second harvest", tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); } } else if (isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector was null. Using default value: {0} samples stored for {1}", maxSamplesStored, - tracker.harvestable.getEndpointMethodName()); + Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector for {0} was null. Using default value: {1} max samples stored per minute", + tracker.harvestable.getEndpointMethodName(), maxSamplesStored); } tracker.start(reportPeriodInMillis, maxSamplesStored); From a8cbbd29a960808713f86fa33e77a027b626f344 Mon Sep 17 00:00:00 2001 From: Todd W Crone Date: Fri, 18 Mar 2022 11:10:17 -0400 Subject: [PATCH 90/96] Fix ClassCastException on large max_samples_stored --- .../ApplicationLoggingForwardingConfig.java | 15 ++++++++++++++- .../ApplicationLoggingForwardingConfigTest.java | 11 +++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java index 5e34240e17..b08d1cae0c 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java @@ -7,7 +7,10 @@ package com.newrelic.agent.config; +import com.newrelic.agent.Agent; + import java.util.Map; +import java.util.logging.Level; public class ApplicationLoggingForwardingConfig extends BaseConfig { public static final String ROOT = "forwarding"; @@ -22,11 +25,21 @@ public class ApplicationLoggingForwardingConfig extends BaseConfig { public ApplicationLoggingForwardingConfig(Map props, String parentRoot, boolean highSecurity) { super(props, parentRoot + ROOT + "."); - maxSamplesStored = getProperty(MAX_SAMPLES_STORED, DEFAULT_MAX_SAMPLES_STORED); + maxSamplesStored = initMaxSamplesStored(); boolean storedMoreThan0 = maxSamplesStored > 0; enabled = storedMoreThan0 && !highSecurity && getProperty(ENABLED, DEFAULT_ENABLED); } + private int initMaxSamplesStored() { + try { + return getProperty(MAX_SAMPLES_STORED, DEFAULT_MAX_SAMPLES_STORED); + } catch (ClassCastException classCastException) { + Agent.LOG.log(Level.WARNING, "The max_samples_stored was likely too large {0}, we will use default {1}", + getProperty(MAX_SAMPLES_STORED), DEFAULT_MAX_SAMPLES_STORED); + return DEFAULT_MAX_SAMPLES_STORED; + } + } + public boolean getEnabled() { return enabled; } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java index cf59b54d20..3fcd263c17 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java @@ -5,6 +5,7 @@ import org.junit.Rule; import org.junit.Test; +import java.math.BigInteger; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -40,6 +41,16 @@ public void testMaxSamplesStoredDefaultValue() { assertEquals(ApplicationLoggingForwardingConfig.DEFAULT_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); } + @Test + public void testMaxSamplesStoredDefaultValueIfValueTooLargeForInteger() { + Map maxSamplesStoreTooLarge = new HashMap<>(localProps); + maxSamplesStoreTooLarge.put(ApplicationLoggingForwardingConfig.MAX_SAMPLES_STORED, new BigInteger("9999999999999999999999")); + + ApplicationLoggingForwardingConfig config = new ApplicationLoggingForwardingConfig(maxSamplesStoreTooLarge, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + false); + assertEquals(ApplicationLoggingForwardingConfig.DEFAULT_MAX_SAMPLES_STORED, config.getMaxSamplesStored()); + } + @Test public void testMaxSamplesStoredNotDefaultValue() { localProps.put(ApplicationLoggingForwardingConfig.MAX_SAMPLES_STORED, TEST_MAX_SAMPLES_STORED); From 0be57ed5da2bf84b5e68d1f835f131cd37da37ac Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 21 Mar 2022 12:32:49 -0700 Subject: [PATCH 91/96] Update application_logging stanza of default yaml --- newrelic-agent/src/main/resources/newrelic.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 573432500c..d3fd3e02f9 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -95,15 +95,15 @@ common: &default_settings # The agent will automatically forward application logs to New Relic in # a format that includes agent metadata for linking them to traces and errors. - forwarding: + #forwarding: # When true, application logs will be forwarded to New Relic. The default is false. - enabled: false + #enabled: false # Application log events are collected up to the configured amount. Afterwards, # events are sampled to maintain an even distribution across the harvest cycle. # Default is 10000. Setting to 0 will disable. - max_samples_stored: 10000 + #max_samples_stored: 10000 # The agent will generate metrics to indicate the number of # application log events occurring at each distinct log level. @@ -115,10 +115,10 @@ common: &default_settings # The agent will add linking metadata to each log line in your application log files. # This feature should only be used if you want to use a third party log forwarder, instead # of the agent's built-in forwarding feature, to send your application log events to New Relic. - local_decorating: + #local_decorating: # When true, the agent will decorate your application log files with linking metadata. The default is false. - enabled: false + #enabled: false # Proxy settings for connecting to the New Relic server: # If a proxy is used, the host setting is required. Other settings From 8dd56a83408249f061d1d385426f8f2fd2536ab6 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Mon, 21 Mar 2022 14:27:29 -0700 Subject: [PATCH 92/96] Add entity.name to LogEvents. Fix tests. --- .../nr/agent/instrumentation/log4j1/AgentUtil.java | 2 ++ .../nr/agent/instrumentation/log4j2/AgentUtil.java | 2 ++ .../java/com/nr/instrumentation/jul/AgentUtil.java | 2 ++ .../instrumentation/logbackclassic12/AgentUtil.java | 2 ++ .../com/newrelic/agent/AgentLinkingMetadata.java | 8 ++++++-- .../com/newrelic/agent/AgentLinkingMetadataTest.java | 12 ++++++------ 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index 2a8f4ee2d3..b1743736ee 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -24,6 +24,7 @@ public class AgentUtil { private static final String TRACE_ID = "trace.id"; private static final String HOSTNAME = "hostname"; private static final String ENTITY_GUID = "entity.guid"; + private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; @@ -47,6 +48,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); } return blob.toString(); } diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 684323f932..7dbc1e5f22 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -29,6 +29,7 @@ public class AgentUtil { private static final String TRACE_ID = "trace.id"; private static final String HOSTNAME = "hostname"; private static final String ENTITY_GUID = "entity.guid"; + private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; @@ -85,6 +86,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); } return blob.toString(); } diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 3db291bb37..25227ce577 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -24,6 +24,7 @@ public class AgentUtil { private static final String TRACE_ID = "trace.id"; private static final String HOSTNAME = "hostname"; private static final String ENTITY_GUID = "entity.guid"; + private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; @@ -47,6 +48,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); } return blob.toString(); } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 7661943dc3..d9fff98be5 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -29,6 +29,7 @@ public class AgentUtil { private static final String TRACE_ID = "trace.id"; private static final String HOSTNAME = "hostname"; private static final String ENTITY_GUID = "entity.guid"; + private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; @@ -76,6 +77,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); + appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); } return blob.toString(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java b/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java index c3c994d44a..0b17785210 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/AgentLinkingMetadata.java @@ -62,8 +62,8 @@ public static Map getLinkingMetadata(TraceMetadata traceMetadata } /** - * Get a map of agent linking metadata minus entity.type, - * entity.name, and any attributes with an empty value. + * Get a map of agent linking metadata minus + * entity.type and any attributes with an empty value. * This subset of linking metadata is added to LogEvents. * * @param traceMetadata TraceMetadataImpl to get spanId and traceId @@ -94,6 +94,10 @@ public static Map getLogEventLinkingMetadata(TraceMetadata trace if (!entityGuid.isEmpty()) { logEventLinkingMetadata.put(ENTITY_GUID, entityGuid); } + String entityName = getEntityName(agentConfig); + if (!entityName.isEmpty()) { + logEventLinkingMetadata.put(ENTITY_NAME, entityName); + } } catch (NullPointerException ignored) { logWarning(); } diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java index 81bcf6716a..adce108d9c 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/AgentLinkingMetadataTest.java @@ -143,8 +143,8 @@ public void getLogEventLinkingMetadata() { // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); - assertFalse("entity.name should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_NAME)); - assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_TYPE)); + assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsKey(AgentLinkingMetadata.ENTITY_TYPE)); + assertEquals(expectedEntityName, linkingMetadata.get(AgentLinkingMetadata.ENTITY_NAME)); assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); assertEquals(expectedTraceId, linkingMetadata.get(AgentLinkingMetadata.TRACE_ID)); @@ -187,12 +187,12 @@ public void getLogEventLinkingMetadataWithEmptyTraceAttributes() { // Can't assert on a specific hostname value as it will resolve to the actual hostname of the machine running the test assertFalse("hostname shouldn't be empty", linkingMetadata.get(AgentLinkingMetadata.HOSTNAME).isEmpty()); - assertFalse("entity.name should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_NAME)); - assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.ENTITY_TYPE)); + assertFalse("entity.type should not be included in LogEvent linking metadata", linkingMetadata.containsKey(AgentLinkingMetadata.ENTITY_TYPE)); + assertEquals(expectedEntityName, linkingMetadata.get(AgentLinkingMetadata.ENTITY_NAME)); assertEquals(expectedEntityGuid, linkingMetadata.get(AgentLinkingMetadata.ENTITY_GUID)); // trace.id and span.id would be empty values if getLogEventLinkingMetadata was called outside of a transaction, in which case they are omitted - assertFalse("empty trace.id value should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.TRACE_ID)); - assertFalse("empty span.id value should not be included in LogEvent linking metadata", linkingMetadata.containsValue(AgentLinkingMetadata.SPAN_ID)); + assertFalse("empty trace.id value should not be included in LogEvent linking metadata", linkingMetadata.containsKey(AgentLinkingMetadata.TRACE_ID)); + assertFalse("empty span.id value should not be included in LogEvent linking metadata", linkingMetadata.containsKey(AgentLinkingMetadata.SPAN_ID)); } } From f2a90979d964c1d398b5aacbe6fa552a3b1351e5 Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Mar 2022 10:39:52 -0700 Subject: [PATCH 93/96] URL encode entity.name for local_decorating metadata blob --- .../instrumentation/log4j1/AgentUtil.java | 21 ++++++++++++++++- .../instrumentation/log4j1/AgentUtilTest.java | 23 +++++++++++++++++++ .../instrumentation/log4j2/AgentUtil.java | 20 +++++++++++++++- .../instrumentation/log4j2/AgentUtilTest.java | 22 ++++++++++++++++++ .../com/nr/instrumentation/jul/AgentUtil.java | 21 ++++++++++++++++- .../nr/instrumentation/jul/AgentUtilTest.java | 22 ++++++++++++++++++ .../logbackclassic12/AgentUtil.java | 20 +++++++++++++++- .../logbackclassic12/AgentUtilTest.java | 22 ++++++++++++++++++ 8 files changed, 167 insertions(+), 4 deletions(-) create mode 100644 instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java create mode 100644 instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java create mode 100644 instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java create mode 100644 instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index b1743736ee..8af3042971 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -9,7 +9,11 @@ import com.newrelic.api.agent.NewRelic; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.logging.Level; public class AgentUtil { public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; @@ -48,7 +52,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); + appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); } return blob.toString(); } @@ -60,6 +64,21 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } + /** + * URL encode a String value. + * + * @param value String to encode + * @return URL encoded String + */ + static String urlEncode(String value) { + try { + value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + NewRelic.getAgent().getLogger().log(Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); + } + return value; + } + /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java new file mode 100644 index 0000000000..88a1e9aac7 --- /dev/null +++ b/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java @@ -0,0 +1,23 @@ +package com.nr.agent.instrumentation.log4j1; + + +import org.junit.Assert; +import org.junit.Test; + +public class AgentUtilTest { + + @Test + public void testUrlEncoding() { + final String ENCODED_PIPE = "%7C"; + final String ENCODED_SPACE = "+"; + // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as + // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files + final String valueToEncode = "|My Application|"; + final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; + + String encodedValue = AgentUtil.urlEncode(valueToEncode); + + Assert.assertEquals(expectedEncodedValue, encodedValue); + } + +} diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 7dbc1e5f22..7eb4e25d83 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -13,6 +13,9 @@ import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.message.Message; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -86,7 +89,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); + appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); } return blob.toString(); } @@ -98,6 +101,21 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } + /** + * URL encode a String value. + * + * @param value String to encode + * @return URL encoded String + */ + static String urlEncode(String value) { + try { + value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + NewRelic.getAgent().getLogger().log(java.util.logging.Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); + } + return value; + } + /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java new file mode 100644 index 0000000000..4a6a5984ad --- /dev/null +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java @@ -0,0 +1,22 @@ +package com.nr.agent.instrumentation.log4j2; + +import org.junit.Assert; +import org.junit.Test; + +public class AgentUtilTest { + + @Test + public void testUrlEncoding() { + final String ENCODED_PIPE = "%7C"; + final String ENCODED_SPACE = "+"; + // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as + // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files + final String valueToEncode = "|My Application|"; + final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; + + String encodedValue = AgentUtil.urlEncode(valueToEncode); + + Assert.assertEquals(expectedEncodedValue, encodedValue); + } + +} diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 25227ce577..d2a4143985 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -9,7 +9,11 @@ import com.newrelic.api.agent.NewRelic; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.logging.Level; public class AgentUtil { public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; @@ -48,7 +52,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); + appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); } return blob.toString(); } @@ -60,6 +64,21 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } + /** + * URL encode a String value. + * + * @param value String to encode + * @return URL encoded String + */ + static String urlEncode(String value) { + try { + value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + NewRelic.getAgent().getLogger().log(Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); + } + return value; + } + /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java b/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java new file mode 100644 index 0000000000..87dff10ece --- /dev/null +++ b/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java @@ -0,0 +1,22 @@ +package com.nr.instrumentation.jul; + +import org.junit.Assert; +import org.junit.Test; + +public class AgentUtilTest { + + @Test + public void testUrlEncoding() { + final String ENCODED_PIPE = "%7C"; + final String ENCODED_SPACE = "+"; + // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as + // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files + final String valueToEncode = "|My Application|"; + final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; + + String encodedValue = AgentUtil.urlEncode(valueToEncode); + + Assert.assertEquals(expectedEncodedValue, encodedValue); + } + +} diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index d9fff98be5..50f8f31ae8 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -11,6 +11,9 @@ import com.newrelic.agent.bridge.AgentBridge; import com.newrelic.api.agent.NewRelic; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -77,7 +80,7 @@ public static String getLinkingMetadataBlob() { appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_NAME), blob); + appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); } return blob.toString(); } @@ -89,6 +92,21 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) blob.append(BLOB_DELIMITER); } + /** + * URL encode a String value. + * + * @param value String to encode + * @return URL encoded String + */ + static String urlEncode(String value) { + try { + value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); + } catch (UnsupportedEncodingException e) { + NewRelic.getAgent().getLogger().log(java.util.logging.Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); + } + return value; + } + /** * Check if all application_logging features are enabled. * diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java new file mode 100644 index 0000000000..31a627f9ad --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java @@ -0,0 +1,22 @@ +package com.nr.agent.instrumentation.logbackclassic12; + +import org.junit.Assert; +import org.junit.Test; + +public class AgentUtilTest { + + @Test + public void testUrlEncoding() { + final String ENCODED_PIPE = "%7C"; + final String ENCODED_SPACE = "+"; + // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as + // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files + final String valueToEncode = "|My Application|"; + final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; + + String encodedValue = AgentUtil.urlEncode(valueToEncode); + + Assert.assertEquals(expectedEncodedValue, encodedValue); + } + +} From b785e48585112dbeb768284149c6366123226c8a Mon Sep 17 00:00:00 2001 From: jasonjkeller Date: Wed, 23 Mar 2022 15:08:43 -0700 Subject: [PATCH 94/96] PR feedback --- .../java/com/nr/agent/instrumentation/log4j1/AgentUtil.java | 2 +- .../java/com/nr/agent/instrumentation/log4j2/AgentUtil.java | 2 +- .../src/main/java/com/nr/instrumentation/jul/AgentUtil.java | 2 +- .../nr/agent/instrumentation/logbackclassic12/AgentUtil.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java index b1743736ee..0a5bf69543 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java @@ -34,7 +34,7 @@ public class AgentUtil { /** * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id| + * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| * * @return agent linking metadata string blob */ diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 7dbc1e5f22..9e4ab964b5 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -72,7 +72,7 @@ public static void recordNewRelicLogEvent(LogEvent event) { /** * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id| + * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| * * @return agent linking metadata string blob */ diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java index 25227ce577..353b91fc39 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java @@ -34,7 +34,7 @@ public class AgentUtil { /** * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id| + * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| * * @return agent linking metadata string blob */ diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index d9fff98be5..78b8b1166c 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -63,7 +63,7 @@ public static void recordNewRelicLogEvent(String message, long timeStampMillis, /** * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id| + * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| * * @return agent linking metadata string blob */ From 67cd317864b81cc92e85c0549deec15acd2f1af6 Mon Sep 17 00:00:00 2001 From: tbradellis Date: Thu, 24 Mar 2022 09:49:59 -0700 Subject: [PATCH 95/96] clarify maxSamples set by --- .../newrelic/agent/HarvestServiceImpl.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java index baa07a1a89..2f478e5257 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/HarvestServiceImpl.java @@ -24,13 +24,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; -import static com.newrelic.agent.config.SpanEventsConfig.*; -import static com.newrelic.agent.transport.CollectorMethods.*; +import static com.newrelic.agent.config.SpanEventsConfig.SERVER_SPAN_HARVEST_CONFIG; +import static com.newrelic.agent.config.SpanEventsConfig.SERVER_SPAN_HARVEST_LIMIT; +import static com.newrelic.agent.transport.CollectorMethods.SPAN_EVENT_DATA; /** * This class is responsible for running the harvest tasks. There is one harvest task per RPM service. A harvest task @@ -100,8 +107,8 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { // The event_harvest_config.harvest_limits received from server-side during the connect lifecycle // contains config for error_event_data, analytic_event_data, custom_event_data, and log_event_data if (eventHarvestConfig != null && !isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "event_harvest_config from collector for {0} is: {1} max samples stored per minute", - tracker.harvestable.getEndpointMethodName(), maxSamplesStored); + Agent.LOG.log(Level.FINE, "event_harvest_config from collector for {0} is: {1} max samples stored per minute", + tracker.harvestable.getEndpointMethodName(), maxSamplesStored); Map harvestLimits = (Map) eventHarvestConfig.get(HARVEST_LIMITS); Long harvestLimit = (Long) harvestLimits.get(tracker.harvestable.getEndpointMethodName()); @@ -109,11 +116,14 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { maxSamplesStored = harvestLimit.intValue(); reportPeriodInMillis = (long) eventHarvestConfig.get(REPORT_PERIOD_MS); // faster event harvest report period float reportPeriodInSeconds = reportPeriodInMillis / 1000; + if (maxSamplesStored == 0) { + Agent.LOG.log(Level.INFO, "harvest limit has been disabled by the collector for {0}", tracker.harvestable.getEndpointMethodName()); + } Agent.LOG.log(Level.FINE, "harvest limit from collector for {0} is: {1} max samples stored per every {2} second harvest", tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); ServiceFactory.getStatsService().doStatsWork( StatsWorks.getRecordMetricWork(MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS, reportPeriodInSeconds), - MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS ); + MetricNames.SUPPORTABILITY_EVENT_HARVEST_REPORT_PERIOD_IN_SECONDS); } } else if (!isSpanEventEndpoint) { Agent.LOG.log(Level.FINE, "event_harvest_config from collector for {0} was null. Using default value: {1} max samples stored per minute", @@ -133,7 +143,8 @@ public void startHarvestables(IRPMService rpmService, AgentConfig config) { tracker.harvestable.getEndpointMethodName(), harvestLimit, reportPeriodInSeconds); } } else if (isSpanEventEndpoint) { - Agent.LOG.log(Level.FINE, "span_event_harvest_config from collector for {0} was null. Using default value: {1} max samples stored per minute", + Agent.LOG.log(Level.FINE, + "span_event_harvest_config from collector for {0} was null. Using default value: {1} max samples stored per minute", tracker.harvestable.getEndpointMethodName(), maxSamplesStored); } @@ -239,7 +250,7 @@ private ScheduledFuture scheduleHarvestTask(HarvestTask harvestTask) { /** * Get the initial delay in milliseconds. - * + *

* Tests can override. */ public long getInitialDelay() { @@ -258,7 +269,7 @@ public void setInitialDelayMillis(long millis) { /** * Get the reporting period in milliseconds. - * + *

* Tests can override. */ public long getReportingPeriod() { @@ -267,7 +278,7 @@ public long getReportingPeriod() { /** * Get the minimum harvest interval in nanoseconds. - * + *

* Tests can override. */ public long getMinHarvestInterval() { @@ -496,7 +507,8 @@ public void run() { } }; - tasks.add(scheduledFasterHarvestExecutor.scheduleAtFixedRate(SafeWrappers.safeRunnable(harvestTask), 0 , reportPeriodInMillis, TimeUnit.MILLISECONDS)); + tasks.add( + scheduledFasterHarvestExecutor.scheduleAtFixedRate(SafeWrappers.safeRunnable(harvestTask), 0, reportPeriodInMillis, TimeUnit.MILLISECONDS)); } public synchronized void stop() { From 9d91a20a2360bd629ef64c3bb68d3b50dc911cdc Mon Sep 17 00:00:00 2001 From: Andre Onuki Date: Thu, 24 Mar 2022 18:10:32 -0400 Subject: [PATCH 96/96] Removing unnecessary call to MessageFormat --- .../newrelic/agent/service/logging/LogSenderServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index cd9e5e9c60..44feb0b67e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -569,7 +569,7 @@ public void recordLogEvent(Map attributes) { LogEvent event = createValidatedEvent(attributes); if (events.offer(event)) { - Agent.LOG.finest(MessageFormat.format("Added event of type {0} in Transaction.", LOG_EVENT_TYPE)); + Agent.LOG.log(Level.FINEST, "Added event of type {0} in Transaction.", LOG_EVENT_TYPE); } else { // Too many events are cached on the transaction, send directly to the reservoir. String applicationName = ServiceFactory.getRPMService().getApplicationName();