From 1d073045a5cdbc042b23d0d5824d5943d206dcc4 Mon Sep 17 00:00:00 2001 From: Stanislav Kubik Date: Fri, 27 May 2022 16:10:04 +0200 Subject: [PATCH 1/2] feature/mdc-in-log-forwarding: Include MDC data into logs forwarded to NR from log4j-2 and logback --- .../instrumentation/log4j2/AgentUtil.java | 23 ++++++++++++++++++ .../spi/LoggingEvent_Instrumentation.java | 8 ++++++- .../logbackclassic12/AgentUtil.java | 24 ++++++++++++++++++- .../src/main/resources/newrelic.yml | 6 +++++ .../com/newrelic/agent/config/newrelic.yml | 2 ++ 5 files changed, 61 insertions(+), 2 deletions(-) diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index d38d996f8f..952bcf89d3 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.util.ReadOnlyStringMap; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -42,11 +43,14 @@ public class AgentUtil { private static final String ENTITY_GUID = "entity.guid"; private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; + // Log attribute prefixes + private static final String MDC_ATTRIBUTE_PREFIX = "mdc."; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED = false; /** * Record a LogEvent to be sent to New Relic. @@ -68,6 +72,10 @@ public static void recordNewRelicLogEvent(LogEvent event) { } logEventMap.put(TIMESTAMP, event.getTimeMillis()); + if (isApplicationLoggingForwardingIncludeMdcEnabled() && event.getContextData() != null) { + event.getContextData().forEach((key, value) -> logEventMap.put(MDC_ATTRIBUTE_PREFIX + key, value)); + } + Level level = event.getLevel(); if (level != null) { String levelName = level.name(); @@ -126,6 +134,12 @@ private static boolean shouldCreateLogEvent(Message message, Throwable throwable return (message != null) || !ExceptionUtil.isThrowableNull(throwable); } + private static int getDefaultNumOfLogEventAttributes(ReadOnlyStringMap mdcPropertyMap) { + return isApplicationLoggingForwardingIncludeMdcEnabled() && mdcPropertyMap != null + ? mdcPropertyMap.size() + DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES + : DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; + } + /** * Gets a String representing the agent linking metadata in blob format: * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| @@ -208,4 +222,13 @@ public static boolean isApplicationLoggingForwardingEnabled() { public static boolean isApplicationLoggingLocalDecoratingEnabled() { return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } + + /** + * Check if the application_logging forwarding include_mdc feature is enabled. + * + * @return true if enabled, else false + */ + public static boolean isApplicationLoggingForwardingIncludeMdcEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.include_mdc.enabled", APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED); + } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index fc4b8c2a5b..c73264789f 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -20,6 +20,8 @@ import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; +import java.util.Map; + @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) public class LoggingEvent_Instrumentation { @@ -71,7 +73,7 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str if (applicationLoggingEnabled && isApplicationLoggingForwardingEnabled()) { // Record and send LogEvent to New Relic - recordNewRelicLogEvent(getFormattedMessage(), timeStamp, level, throwable, threadName, threadId, loggerName, fqnOfLoggerClass); + recordNewRelicLogEvent(getFormattedMessage(), getMdc(), timeStamp, level, throwable, threadName, threadId, loggerName, fqnOfLoggerClass); } } @@ -83,4 +85,8 @@ private Throwable extractThrowableAnRearrangeArguments(Object[] argArray) { return Weaver.callOriginal(); } + public Map getMdc() { + return Weaver.callOriginal(); + } + } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index a3585e90ed..99a36e4206 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -40,11 +40,14 @@ public class AgentUtil { private static final String ENTITY_GUID = "entity.guid"; private static final String ENTITY_NAME = "entity.name"; private static final String SPAN_ID = "span.id"; + // Log attribute prefixes + private static final String MDC_ATTRIBUTE_PREFIX = "mdc."; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED = false; /** * Record a LogEvent to be sent to New Relic. @@ -53,7 +56,7 @@ public class AgentUtil { * @param timeStampMillis log timestamp * @param level log level */ - public static void recordNewRelicLogEvent(String message, long timeStampMillis, Level level, Throwable throwable, String threadName, long threadId, + public static void recordNewRelicLogEvent(String message, Map mdcPropertyMap, long timeStampMillis, Level level, Throwable throwable, String threadName, long threadId, String loggerName, String fqcnLoggerName) { boolean messageEmpty = message.isEmpty(); @@ -65,6 +68,10 @@ public static void recordNewRelicLogEvent(String message, long timeStampMillis, } logEventMap.put(TIMESTAMP, timeStampMillis); + if (isApplicationLoggingForwardingIncludeMdcEnabled()) { + mdcPropertyMap.forEach((key, value) -> logEventMap.put(MDC_ATTRIBUTE_PREFIX + key, value)); + } + if (level.toString().isEmpty()) { logEventMap.put(LEVEL, UNKNOWN); } else { @@ -115,6 +122,12 @@ private static boolean shouldCreateLogEvent(boolean messageEmpty, Throwable thro return !messageEmpty || !ExceptionUtil.isThrowableNull(throwable); } + private static int getDefaultNumOfLogEventAttributes(Map mdcPropertyMap) { + return isApplicationLoggingForwardingIncludeMdcEnabled() + ? mdcPropertyMap.size() + DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES + : DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; + } + /** * Gets a String representing the agent linking metadata in blob format: * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| @@ -197,4 +210,13 @@ public static boolean isApplicationLoggingForwardingEnabled() { public static boolean isApplicationLoggingLocalDecoratingEnabled() { return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } + + /** + * Check if the application_logging forwarding include_mdc feature is enabled. + * + * @return true if enabled, else false + */ + public static boolean isApplicationLoggingForwardingIncludeMdcEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.include_mdc.enabled", APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED); + } } diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 620654dda8..8b22b65094 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -105,6 +105,12 @@ common: &default_settings # Default is 10000. Setting to 0 will disable. #max_samples_stored: 10000 + # Whether the log events should include MDC from loggers with support for that. + include_mdc: + + # When true, application logs will contain MDC data. + enabled: false + # The agent will generate metrics to indicate the number of # application log events occurring at each distinct log level. metrics: diff --git a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml index 710613719e..1d5467afd6 100644 --- a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml +++ b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml @@ -22,6 +22,8 @@ common: &default_settings enabled: true forwarding: enabled: true + include_mdc: + enabled: false max_samples_stored: 10000 metrics: enabled: true From 66bbdecb39d33fea23a08be051e737bf680d90d3 Mon Sep 17 00:00:00 2001 From: Andre Onuki Date: Thu, 15 Sep 2022 16:04:19 -0400 Subject: [PATCH 2/2] NR-23881 - Give users possibility to add MDC to logging Filtering mapped context attributes given include/exclude lists. Extracting common methods from AgentUtils classes to AppLogginUtil in agent-bridge --- .../com/newrelic/agent/bridge/NoOpLogs.java | 5 +- .../agent/bridge/logging/AppLoggingUtils.java | 56 ++++-- .../agent/bridge/logging/LogAttributeKey.java | 52 ++++++ .../bridge/logging/LogAttributeType.java | 28 +++ .../java/com/newrelic/api/agent/Logs.java | 4 +- .../bridge/logging/AppLoggingUtilsTest.java | 20 ++- .../IntrospectorLogSenderService.java | 7 +- .../log4j/Category_Instrumentation.java | 4 +- .../instrumentation/log4j2/AgentUtil.java | 159 +++--------------- .../config/LoggerConfig_Instrumentation.java | 6 +- .../StringBuilderEncoder_Instrumentation.java | 6 +- .../instrumentation/log4j2/AgentUtilTest.java | 22 --- .../log4j2/ContextDataTest.java | 64 +++++++ .../LoggerConfig_InstrumentationTest.java | 21 +++ ...plication_logging_context_data_enabled.yml | 12 ++ .../com/nr/instrumentation/jul/AgentUtil.java | 119 ------------- .../util/logging/Logger_Instrumentation.java | 4 +- .../nr/instrumentation/jul/AgentUtilTest.java | 22 --- .../classic/Logger_Instrumentation.java | 6 +- .../spi/LoggingEvent_Instrumentation.java | 22 ++- .../logbackclassic12/AgentUtil.java | 156 +++-------------- .../logbackclassic12/AgentUtilTest.java | 22 --- .../logbackclassic12/ContextDataTest.java | 63 +++++++ .../LoggingEvent_InstrumentationTest.java | 21 +++ ...plication_logging_context_data_enabled.yml | 12 ++ instrumentation/spring-aop-2/build.gradle | 3 - .../com/newrelic/agent/DummyTransaction.java | 3 +- .../agent/attributes/AttributesNode.java | 2 +- .../DisabledExcludeIncludeFilter.java | 25 +++ .../attributes/ExcludeIncludeFilter.java | 13 ++ .../attributes/ExcludeIncludeFilterImpl.java | 57 +++++++ .../config/ApplicationLoggingConfig.java | 22 +++ .../config/ApplicationLoggingConfigImpl.java | 21 +++ .../ApplicationLoggingContextDataConfig.java | 56 ++++++ .../ApplicationLoggingForwardingConfig.java | 23 +++ .../analytics/InsightsServiceImpl.java | 4 +- .../service/logging/LogSenderServiceImpl.java | 91 +++++++--- .../src/main/resources/newrelic.yml | 12 +- .../ExcludeIncludeFilterImplTest.java | 34 ++++ ...plicationLoggingContextDataConfigTest.java | 112 ++++++++++++ ...pplicationLoggingForwardingConfigTest.java | 1 + .../logging/LogSenderServiceImplTest.java | 47 ++++-- .../com/newrelic/agent/config/newrelic.yml | 2 +- 43 files changed, 889 insertions(+), 552 deletions(-) rename instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java => agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/AppLoggingUtils.java (59%) create mode 100644 agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeKey.java create mode 100644 agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeType.java rename instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java => agent-bridge/src/test/java/com/newrelic/agent/bridge/logging/AppLoggingUtilsTest.java (60%) delete mode 100644 instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java create mode 100644 instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/ContextDataTest.java create mode 100644 instrumentation/apache-log4j-2/src/test/resources/application_logging_context_data_enabled.yml delete mode 100644 instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java delete mode 100644 instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java delete mode 100644 instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java create mode 100644 instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/ContextDataTest.java create mode 100644 instrumentation/logback-classic-1.2/src/test/resources/application_logging_context_data_enabled.yml create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/attributes/DisabledExcludeIncludeFilter.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilter.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImpl.java create mode 100644 newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfig.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImplTest.java create mode 100644 newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfigTest.java diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java index 7825027abc..ea3e436f2b 100644 --- a/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpLogs.java @@ -7,6 +7,7 @@ package com.newrelic.agent.bridge; +import com.newrelic.agent.bridge.logging.LogAttributeKey; import com.newrelic.api.agent.Logs; import java.util.Map; @@ -18,7 +19,7 @@ private NoOpLogs() { } @Override - public void recordLogEvent(Map attributes) { - } + public void recordLogEvent(Map attributes) { + } } diff --git a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/AppLoggingUtils.java similarity index 59% rename from instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java rename to agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/AppLoggingUtils.java index 7e0ebcd48b..8b45f9dc59 100644 --- a/instrumentation/apache-log4j-1/src/main/java/com/nr/agent/instrumentation/log4j1/AgentUtil.java +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/AppLoggingUtils.java @@ -4,37 +4,47 @@ * * SPDX-License-Identifier: Apache-2.0 * */ +package com.newrelic.agent.bridge.logging; -package com.nr.agent.instrumentation.log4j1; - +import com.newrelic.agent.bridge.logging.LogAttributeKey; +import com.newrelic.agent.bridge.logging.LogAttributeType; import com.newrelic.api.agent.NewRelic; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.logging.Level; -public class AgentUtil { - public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; +public class AppLoggingUtils { + public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 10; // Log message attributes - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String LEVEL = "level"; + public static final LogAttributeKey MESSAGE = new LogAttributeKey("message", LogAttributeType.AGENT); + public static final LogAttributeKey TIMESTAMP = new LogAttributeKey("timestamp", LogAttributeType.AGENT); + public static final LogAttributeKey LEVEL = new LogAttributeKey("level", LogAttributeType.AGENT); + public static final LogAttributeKey ERROR_MESSAGE = new LogAttributeKey("error.message", LogAttributeType.AGENT); + public static final LogAttributeKey ERROR_CLASS = new LogAttributeKey("error.class", LogAttributeType.AGENT); + public static final LogAttributeKey ERROR_STACK = new LogAttributeKey("error.stack", LogAttributeType.AGENT); + public static final LogAttributeKey THREAD_NAME = new LogAttributeKey("thread.name", LogAttributeType.AGENT); + public static final LogAttributeKey THREAD_ID = new LogAttributeKey("thread.id", LogAttributeType.AGENT); + public static final LogAttributeKey LOGGER_NAME = new LogAttributeKey("logger.name", LogAttributeType.AGENT); + public static final LogAttributeKey LOGGER_FQCN = new LogAttributeKey("logger.fqcn", LogAttributeType.AGENT); public static final String UNKNOWN = "UNKNOWN"; // Linking metadata attributes used in blob - private static final String BLOB_PREFIX = "NR-LINKING"; - private static final String BLOB_DELIMITER = "|"; - private static final String TRACE_ID = "trace.id"; - private static final String HOSTNAME = "hostname"; - private static final String ENTITY_GUID = "entity.guid"; - private static final String ENTITY_NAME = "entity.name"; - private static final String SPAN_ID = "span.id"; + public static final String BLOB_PREFIX = "NR-LINKING"; + public static final String BLOB_DELIMITER = "|"; + public static final String TRACE_ID = "trace.id"; + public static final String HOSTNAME = "hostname"; + public static final String ENTITY_GUID = "entity.guid"; + public static final String ENTITY_NAME = "entity.name"; + public static final String SPAN_ID = "span.id"; + // Log attribute prefixes + public static final String CONTEXT_DATA_ATTRIBUTE_PREFIX = "context."; // Enabled defaults private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; + private static final boolean APP_LOGGING_FORWARDING_INCLUDE_CONTEXT_DATA_DEFAULT_ENABLED = false; /** * Gets a String representing the agent linking metadata in blob format: @@ -70,13 +80,15 @@ private static void appendAttributeToBlob(String attribute, StringBuilder blob) * @param value String to encode * @return URL encoded String */ - static String urlEncode(String value) { + public static String urlEncode(String value) { try { if (value != null) { value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); } } catch (UnsupportedEncodingException e) { - NewRelic.getAgent().getLogger().log(Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); + NewRelic.getAgent() + .getLogger() + .log(java.util.logging.Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); } return value; } @@ -116,4 +128,14 @@ public static boolean isApplicationLoggingForwardingEnabled() { public static boolean isApplicationLoggingLocalDecoratingEnabled() { return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); } + + /** + * Check if the application_logging forwarding include_context_data feature is enabled. + * + * @return true if enabled, else false + */ + public static boolean isAppLoggingContextDataEnabled() { + return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.context_data.enabled", + APP_LOGGING_FORWARDING_INCLUDE_CONTEXT_DATA_DEFAULT_ENABLED); + } } diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeKey.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeKey.java new file mode 100644 index 0000000000..c1e3fe41b4 --- /dev/null +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeKey.java @@ -0,0 +1,52 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.bridge.logging; + +import java.util.Objects; + +public class LogAttributeKey { + public final String key; + public final LogAttributeType type; + + public LogAttributeKey(String key, LogAttributeType type) { + this.key = key; + this.type = type; + } + + public String getKey() { + return key; + } + + public String getPrefixedKey() { + if (key == null || type == null) { + return key; + } + return type.applyPrefix(key); + } + + public LogAttributeType getType() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogAttributeKey that = (LogAttributeKey) o; + return Objects.equals(key, that.key) && type == that.type; + } + + @Override + public int hashCode() { + return Objects.hash(key, type); + } +} diff --git a/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeType.java b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeType.java new file mode 100644 index 0000000000..4913d8408e --- /dev/null +++ b/agent-bridge/src/main/java/com/newrelic/agent/bridge/logging/LogAttributeType.java @@ -0,0 +1,28 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.bridge.logging; + +public enum LogAttributeType { + AGENT(null) { + @Override + public String applyPrefix(String key) { + return key; + } + }, + CONTEXT(AppLoggingUtils.CONTEXT_DATA_ATTRIBUTE_PREFIX); + + private final String prefix; + + LogAttributeType(String prefix) { + this.prefix = prefix; + } + + public String applyPrefix(String key) { + return prefix.concat(key); + } +} diff --git a/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java index bd8170e9c5..3ab79b0057 100644 --- a/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java +++ b/agent-bridge/src/main/java/com/newrelic/api/agent/Logs.java @@ -7,6 +7,8 @@ package com.newrelic.api.agent; +import com.newrelic.agent.bridge.logging.LogAttributeKey; + import java.util.Map; /** @@ -22,5 +24,5 @@ public interface Logs { * For map values that are not String, Number, or Boolean object types the toString value will be used. * @since 7.6.0 */ - void recordLogEvent(Map attributes); + void recordLogEvent(Map attributes); } diff --git a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java b/agent-bridge/src/test/java/com/newrelic/agent/bridge/logging/AppLoggingUtilsTest.java similarity index 60% rename from instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java rename to agent-bridge/src/test/java/com/newrelic/agent/bridge/logging/AppLoggingUtilsTest.java index 88a1e9aac7..465c2b05bd 100644 --- a/instrumentation/apache-log4j-1/src/test/java/com/nr/agent/instrumentation/log4j1/AgentUtilTest.java +++ b/agent-bridge/src/test/java/com/newrelic/agent/bridge/logging/AppLoggingUtilsTest.java @@ -1,10 +1,19 @@ -package com.nr.agent.instrumentation.log4j1; - +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package com.newrelic.agent.bridge.logging; +import com.newrelic.agent.bridge.logging.AppLoggingUtils; import org.junit.Assert; import org.junit.Test; -public class AgentUtilTest { +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + +public class AppLoggingUtilsTest { @Test public void testUrlEncoding() { @@ -15,9 +24,8 @@ public void testUrlEncoding() { final String valueToEncode = "|My Application|"; final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; - String encodedValue = AgentUtil.urlEncode(valueToEncode); + String encodedValue = AppLoggingUtils.urlEncode(valueToEncode); Assert.assertEquals(expectedEncodedValue, encodedValue); } - -} +} \ No newline at end of file diff --git a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java index 35392fccc1..466d3d809c 100644 --- a/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java +++ b/instrumentation-test/src/main/java/com/newrelic/agent/introspec/internal/IntrospectorLogSenderService.java @@ -8,6 +8,7 @@ package com.newrelic.agent.introspec.internal; import com.newrelic.agent.Agent; +import com.newrelic.agent.bridge.logging.LogAttributeKey; import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.deps.com.google.common.collect.LinkedListMultimap; import com.newrelic.agent.deps.com.google.common.collect.ListMultimap; @@ -23,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.stream.Collectors; import static com.newrelic.agent.model.LogEvent.LOG_EVENT_TYPE; @@ -75,9 +77,10 @@ public boolean isStoppedOrStopping() { } @Override - public void recordLogEvent(Map attributes) { + public void recordLogEvent(Map attributes) { if (AnalyticsEvent.isValidType(LOG_EVENT_TYPE)) { - Map atts = Maps.newHashMap(attributes); + Map atts = attributes.entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey().getPrefixedKey(), Map.Entry::getValue)); LogEvent event = new LogEvent(atts, DistributedTraceServiceImpl.nextTruncatedFloat()); storeEvent("TestApp", event); } diff --git a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java index ddc1e62802..ea8c9764e2 100644 --- a/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java +++ b/instrumentation/apache-log4j-1/src/main/java/org/apache/log4j/Category_Instrumentation.java @@ -11,8 +11,8 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.agent.instrumentation.log4j1.AgentUtil.isApplicationLoggingEnabled; -import static com.nr.agent.instrumentation.log4j1.AgentUtil.isApplicationLoggingMetricsEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingMetricsEnabled; @Weave(originalName = "org.apache.log4j.Category") public class Category_Instrumentation { diff --git a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java index 952bcf89d3..c98bb763dd 100644 --- a/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java +++ b/instrumentation/apache-log4j-2/src/main/java/com/nr/agent/instrumentation/log4j2/AgentUtil.java @@ -8,50 +8,31 @@ package com.nr.agent.instrumentation.log4j2; import com.newrelic.agent.bridge.AgentBridge; -import com.newrelic.api.agent.NewRelic; +import com.newrelic.agent.bridge.logging.AppLoggingUtils; +import com.newrelic.agent.bridge.logging.LogAttributeKey; +import com.newrelic.agent.bridge.logging.LogAttributeType; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.message.Message; import org.apache.logging.log4j.util.ReadOnlyStringMap; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -public class AgentUtil { - public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 10; - // Log message attributes - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String LEVEL = "level"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - public static final String THREAD_NAME = "thread.name"; - public static final String THREAD_ID = "thread.id"; - public static final String LOGGER_NAME = "logger.name"; - public static final String LOGGER_FQCN = "logger.fqcn"; - public static final String UNKNOWN = "UNKNOWN"; - - // Linking metadata attributes used in blob - private static final String BLOB_PREFIX = "NR-LINKING"; - private static final String BLOB_DELIMITER = "|"; - private static final String TRACE_ID = "trace.id"; - private static final String HOSTNAME = "hostname"; - private static final String ENTITY_GUID = "entity.guid"; - private static final String ENTITY_NAME = "entity.name"; - private static final String SPAN_ID = "span.id"; - // Log attribute prefixes - private static final String MDC_ATTRIBUTE_PREFIX = "mdc."; - // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED = false; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_CLASS; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_MESSAGE; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_STACK; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LEVEL; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LOGGER_FQCN; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LOGGER_NAME; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.MESSAGE; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.THREAD_ID; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.THREAD_NAME; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.TIMESTAMP; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.UNKNOWN; +public class AgentUtil { /** * Record a LogEvent to be sent to New Relic. * @@ -63,7 +44,8 @@ public static void recordNewRelicLogEvent(LogEvent event) { Throwable throwable = event.getThrown(); if (shouldCreateLogEvent(message, throwable)) { - HashMap logEventMap = new HashMap<>(DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES); + ReadOnlyStringMap contextData = event.getContextData(); + Map logEventMap = new HashMap<>(calculateInitialMapSize(contextData)); if (message != null) { String formattedMessage = message.getFormattedMessage(); if (formattedMessage != null && !formattedMessage.isEmpty()) { @@ -72,8 +54,13 @@ public static void recordNewRelicLogEvent(LogEvent event) { } logEventMap.put(TIMESTAMP, event.getTimeMillis()); - if (isApplicationLoggingForwardingIncludeMdcEnabled() && event.getContextData() != null) { - event.getContextData().forEach((key, value) -> logEventMap.put(MDC_ATTRIBUTE_PREFIX + key, value)); + if (AppLoggingUtils.isAppLoggingContextDataEnabled() && contextData != null) { + for (Map.Entry entry : contextData.toMap().entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + LogAttributeKey logAttrKey = new LogAttributeKey(key, LogAttributeType.CONTEXT); + logEventMap.put(logAttrKey, value); + } } Level level = event.getLevel(); @@ -134,101 +121,9 @@ private static boolean shouldCreateLogEvent(Message message, Throwable throwable return (message != null) || !ExceptionUtil.isThrowableNull(throwable); } - private static int getDefaultNumOfLogEventAttributes(ReadOnlyStringMap mdcPropertyMap) { - return isApplicationLoggingForwardingIncludeMdcEnabled() && mdcPropertyMap != null + private static int calculateInitialMapSize(ReadOnlyStringMap mdcPropertyMap) { + return AppLoggingUtils.isAppLoggingContextDataEnabled() && mdcPropertyMap != null ? mdcPropertyMap.size() + DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES : DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; } - - /** - * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| - * - * @return agent linking metadata string blob - */ - public static String getLinkingMetadataBlob() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - StringBuilder blob = new StringBuilder(); - blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); - appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); - } - return blob.toString(); - } - - private static void appendAttributeToBlob(String attribute, StringBuilder blob) { - if (attribute != null && !attribute.isEmpty()) { - blob.append(attribute); - } - blob.append(BLOB_DELIMITER); - } - - /** - * URL encode a String value. - * - * @param value String to encode - * @return URL encoded String - */ - static String urlEncode(String value) { - try { - if (value != null) { - value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); - } - } catch (UnsupportedEncodingException e) { - NewRelic.getAgent() - .getLogger() - .log(java.util.logging.Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); - } - return value; - } - - /** - * Check if all application_logging features are enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging metrics feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging forwarding feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging local_decorating feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging forwarding include_mdc feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingForwardingIncludeMdcEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.include_mdc.enabled", APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED); - } } diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java index 4359579be2..4841bcb726 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/config/LoggerConfig_Instrumentation.java @@ -17,9 +17,9 @@ import java.util.concurrent.atomic.AtomicBoolean; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingForwardingEnabled; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingMetricsEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingForwardingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingMetricsEnabled; import static com.nr.agent.instrumentation.log4j2.AgentUtil.recordNewRelicLogEvent; @Weave(originalName = "org.apache.logging.log4j.core.config.LoggerConfig", type = MatchType.ExactClass) diff --git a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java index 3a5810af76..e09c3e19a5 100644 --- a/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java +++ b/instrumentation/apache-log4j-2/src/main/java/org/apache/logging/log4j/core/layout/StringBuilderEncoder_Instrumentation.java @@ -11,9 +11,9 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.getLinkingMetadataBlob; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingEnabled; -import static com.nr.agent.instrumentation.log4j2.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.getLinkingMetadataBlob; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingLocalDecoratingEnabled; @Weave(originalName = "org.apache.logging.log4j.core.layout.StringBuilderEncoder", type = MatchType.BaseClass) public class StringBuilderEncoder_Instrumentation { diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java deleted file mode 100644 index 4a6a5984ad..0000000000 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/AgentUtilTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.nr.agent.instrumentation.log4j2; - -import org.junit.Assert; -import org.junit.Test; - -public class AgentUtilTest { - - @Test - public void testUrlEncoding() { - final String ENCODED_PIPE = "%7C"; - final String ENCODED_SPACE = "+"; - // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as - // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files - final String valueToEncode = "|My Application|"; - final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; - - String encodedValue = AgentUtil.urlEncode(valueToEncode); - - Assert.assertEquals(expectedEncodedValue, encodedValue); - } - -} diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/ContextDataTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/ContextDataTest.java new file mode 100644 index 0000000000..09b8893aec --- /dev/null +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/ContextDataTest.java @@ -0,0 +1,64 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package com.nr.agent.instrumentation.log4j2; + +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.Introspector; +import com.newrelic.agent.model.LogEvent; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; +import org.apache.logging.log4j.core.config.Configurator; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.Collection; +import java.util.Map; + +import static org.junit.Assert.*; + +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "org.apache.logging.log4j.core" }, configName = "application_logging_context_data_enabled.yml") +public class ContextDataTest { + + private final Introspector introspector = InstrumentationTestRunner.getIntrospector(); + + @Before + public void reset() { + Configurator.reconfigure(); + introspector.clearLogEvents(); + } + + @Test + public void testAttributes() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + String attrKey1 = "key"; + String attrKey2 = "anotherKey"; + String val1 = "42"; + String val2 = "panic"; + + ThreadContext.put(attrKey1, val1); + ThreadContext.put(attrKey2, val2); + logger.error("message"); + + Collection logEvents = introspector.getLogEvents(); + assertEquals(1, logEvents.size()); + + LogEvent logEvent = logEvents.iterator().next(); + Map attributes = logEvent.getUserAttributesCopy(); + long contextAttrCount = attributes.keySet().stream() + .filter(key -> key.startsWith("context.")) + .count(); + // MDC data is filtered later in the process + assertEquals(2L, contextAttrCount); + assertEquals(val1, attributes.get("context." + attrKey1)); + assertEquals(val2, attributes.get("context." + attrKey2)); + } + +} diff --git a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java index df15051052..5fc7516329 100644 --- a/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java +++ b/instrumentation/apache-log4j-2/src/test/java/com/nr/agent/instrumentation/log4j2/LoggerConfig_InstrumentationTest.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LoggerContext; @@ -25,6 +26,7 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @RunWith(InstrumentationTestRunner.class) @@ -598,4 +600,23 @@ private void setLoggerLevel(Level level) { context.updateLoggers(); } + @Test + public void contextDataDisabledTest() { + final Logger logger = LogManager.getLogger(LoggerConfig_InstrumentationTest.class); + ThreadContext.put("include", "42"); + ThreadContext.put("exclude", "panic"); + + logger.error("message"); + + Collection logEvents = introspector.getLogEvents(); + assertEquals(1, logEvents.size()); + + // verify no context attrs + Map attributes = logEvents.iterator().next().getUserAttributesCopy(); + long contextAttrCount = attributes.keySet().stream() + .filter(key -> key.startsWith("context.")) + .count(); + assertEquals(0L, contextAttrCount); + } + } diff --git a/instrumentation/apache-log4j-2/src/test/resources/application_logging_context_data_enabled.yml b/instrumentation/apache-log4j-2/src/test/resources/application_logging_context_data_enabled.yml new file mode 100644 index 0000000000..1212edc2e9 --- /dev/null +++ b/instrumentation/apache-log4j-2/src/test/resources/application_logging_context_data_enabled.yml @@ -0,0 +1,12 @@ +common: &default_settings + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 10000 + context_data: + enabled: true + metrics: + enabled: true + local_decorating: + enabled: false diff --git a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java b/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java deleted file mode 100644 index ff03df5b5a..0000000000 --- a/instrumentation/java.logging-jdk8/src/main/java/com/nr/instrumentation/jul/AgentUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * - * * Copyright 2022 New Relic Corporation. All rights reserved. - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package com.nr.instrumentation.jul; - -import com.newrelic.api.agent.NewRelic; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.logging.Level; - -public class AgentUtil { - public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 3; - // Log message attributes - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String LEVEL = "level"; - public static final String UNKNOWN = "UNKNOWN"; - // Linking metadata attributes used in blob - private static final String BLOB_PREFIX = "NR-LINKING"; - private static final String BLOB_DELIMITER = "|"; - private static final String TRACE_ID = "trace.id"; - private static final String HOSTNAME = "hostname"; - private static final String ENTITY_GUID = "entity.guid"; - private static final String ENTITY_NAME = "entity.name"; - private static final String SPAN_ID = "span.id"; - // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; - - /** - * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| - * - * @return agent linking metadata string blob - */ - public static String getLinkingMetadataBlob() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - StringBuilder blob = new StringBuilder(); - blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); - appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); - } - return blob.toString(); - } - - private static void appendAttributeToBlob(String attribute, StringBuilder blob) { - if (attribute != null && !attribute.isEmpty()) { - blob.append(attribute); - } - blob.append(BLOB_DELIMITER); - } - - /** - * URL encode a String value. - * - * @param value String to encode - * @return URL encoded String - */ - static String urlEncode(String value) { - try { - if (value != null) { - value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); - } - } catch (UnsupportedEncodingException e) { - NewRelic.getAgent().getLogger().log(Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); - } - return value; - } - - /** - * Check if all application_logging features are enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging metrics feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging forwarding feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging local_decorating feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - } -} diff --git a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java index bef11fa70f..56051560c4 100644 --- a/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java +++ b/instrumentation/java.logging-jdk8/src/main/java/java/util/logging/Logger_Instrumentation.java @@ -11,8 +11,8 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.instrumentation.jul.AgentUtil.isApplicationLoggingEnabled; -import static com.nr.instrumentation.jul.AgentUtil.isApplicationLoggingMetricsEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingMetricsEnabled; @Weave(originalName = "java.util.logging.Logger") public class Logger_Instrumentation { diff --git a/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java b/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java deleted file mode 100644 index 87dff10ece..0000000000 --- a/instrumentation/java.logging-jdk8/src/test/java/com/nr/instrumentation/jul/AgentUtilTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.nr.instrumentation.jul; - -import org.junit.Assert; -import org.junit.Test; - -public class AgentUtilTest { - - @Test - public void testUrlEncoding() { - final String ENCODED_PIPE = "%7C"; - final String ENCODED_SPACE = "+"; - // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as - // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files - final String valueToEncode = "|My Application|"; - final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; - - String encodedValue = AgentUtil.urlEncode(valueToEncode); - - Assert.assertEquals(expectedEncodedValue, encodedValue); - } - -} diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java index b817526a65..cf39728533 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/Logger_Instrumentation.java @@ -7,13 +7,13 @@ package ch.qos.logback.classic; +import com.newrelic.agent.bridge.logging.AppLoggingUtils; import com.newrelic.api.agent.NewRelic; import com.newrelic.api.agent.weaver.MatchType; import com.newrelic.api.agent.weaver.NewField; import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.WeaveAllConstructors; import com.newrelic.api.agent.weaver.Weaver; -import com.nr.agent.instrumentation.logbackclassic12.AgentUtil; import org.slf4j.Marker; import java.util.concurrent.atomic.AtomicBoolean; @@ -34,8 +34,8 @@ public abstract class Logger_Instrumentation { private void buildLoggingEventAndAppend(final String localFQCN, final Marker marker, final Level level, final String msg, final Object[] params, final Throwable t) { // Do nothing if application_logging.enabled: false - if (AgentUtil.isApplicationLoggingEnabled()) { - if (AgentUtil.isApplicationLoggingMetricsEnabled()) { + if (AppLoggingUtils.isApplicationLoggingEnabled()) { + if (AppLoggingUtils.isApplicationLoggingMetricsEnabled()) { // Generate log level metrics NewRelic.incrementCounter("Logging/lines"); NewRelic.incrementCounter("Logging/lines/" + level); diff --git a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java index c73264789f..4ebf7725ac 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java +++ b/instrumentation/logback-classic-1.2/src/main/java/ch/qos/logback/classic/spi/LoggingEvent_Instrumentation.java @@ -14,14 +14,16 @@ import com.newrelic.api.agent.weaver.Weave; import com.newrelic.api.agent.weaver.Weaver; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.getLinkingMetadataBlob; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingEnabled; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingForwardingEnabled; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.isApplicationLoggingLocalDecoratingEnabled; -import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; - +import java.util.Collections; import java.util.Map; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.getLinkingMetadataBlob; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isAppLoggingContextDataEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingForwardingEnabled; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.isApplicationLoggingLocalDecoratingEnabled; +import static com.nr.agent.instrumentation.logbackclassic12.AgentUtil.recordNewRelicLogEvent; + @Weave(originalName = "ch.qos.logback.classic.spi.LoggingEvent", type = MatchType.ExactClass) public class LoggingEvent_Instrumentation { @@ -72,8 +74,14 @@ public LoggingEvent_Instrumentation(String fqcn, Logger logger, Level level, Str long threadId = thread.getId(); if (applicationLoggingEnabled && isApplicationLoggingForwardingEnabled()) { + Map mdc; + if (isAppLoggingContextDataEnabled()) { + mdc = getMdc(); + } else { + mdc = Collections.emptyMap(); + } // Record and send LogEvent to New Relic - recordNewRelicLogEvent(getFormattedMessage(), getMdc(), timeStamp, level, throwable, threadName, threadId, loggerName, fqnOfLoggerClass); + recordNewRelicLogEvent(getFormattedMessage(), mdc, timeStamp, level, throwable, threadName, threadId, loggerName, fqnOfLoggerClass); } } diff --git a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java index 99a36e4206..12b6fc61f0 100644 --- a/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java +++ b/instrumentation/logback-classic-1.2/src/main/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtil.java @@ -9,46 +9,27 @@ import ch.qos.logback.classic.Level; import com.newrelic.agent.bridge.AgentBridge; -import com.newrelic.api.agent.NewRelic; +import com.newrelic.agent.bridge.logging.AppLoggingUtils; +import com.newrelic.agent.bridge.logging.LogAttributeKey; +import com.newrelic.agent.bridge.logging.LogAttributeType; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -public class AgentUtil { - public static final int DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES = 10; - // Log message attributes - public static final String MESSAGE = "message"; - public static final String TIMESTAMP = "timestamp"; - public static final String LEVEL = "level"; - public static final String ERROR_MESSAGE = "error.message"; - public static final String ERROR_CLASS = "error.class"; - public static final String ERROR_STACK = "error.stack"; - public static final String THREAD_NAME = "thread.name"; - public static final String THREAD_ID = "thread.id"; - public static final String LOGGER_NAME = "logger.name"; - public static final String LOGGER_FQCN = "logger.fqcn"; - public static final String UNKNOWN = "UNKNOWN"; - - // Linking metadata attributes used in blob - private static final String BLOB_PREFIX = "NR-LINKING"; - private static final String BLOB_DELIMITER = "|"; - private static final String TRACE_ID = "trace.id"; - private static final String HOSTNAME = "hostname"; - private static final String ENTITY_GUID = "entity.guid"; - private static final String ENTITY_NAME = "entity.name"; - private static final String SPAN_ID = "span.id"; - // Log attribute prefixes - private static final String MDC_ATTRIBUTE_PREFIX = "mdc."; - // Enabled defaults - private static final boolean APP_LOGGING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_METRICS_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_FORWARDING_DEFAULT_ENABLED = true; - private static final boolean APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED = false; - private static final boolean APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED = false; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_CLASS; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_MESSAGE; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.ERROR_STACK; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LEVEL; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LOGGER_FQCN; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.LOGGER_NAME; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.MESSAGE; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.THREAD_ID; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.THREAD_NAME; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.TIMESTAMP; +import static com.newrelic.agent.bridge.logging.AppLoggingUtils.UNKNOWN; +public class AgentUtil { /** * Record a LogEvent to be sent to New Relic. * @@ -61,15 +42,18 @@ public static void recordNewRelicLogEvent(String message, Map md boolean messageEmpty = message.isEmpty(); if (shouldCreateLogEvent(messageEmpty, throwable)) { - HashMap logEventMap = new HashMap<>(DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES); + Map logEventMap = new HashMap<>(calculateInitialMapSize(mdcPropertyMap)); if (!messageEmpty) { logEventMap.put(MESSAGE, message); } logEventMap.put(TIMESTAMP, timeStampMillis); - if (isApplicationLoggingForwardingIncludeMdcEnabled()) { - mdcPropertyMap.forEach((key, value) -> logEventMap.put(MDC_ATTRIBUTE_PREFIX + key, value)); + if (AppLoggingUtils.isAppLoggingContextDataEnabled()) { + for (Map.Entry mdcEntry : mdcPropertyMap.entrySet()) { + LogAttributeKey logAttrKey = new LogAttributeKey(mdcEntry.getKey(), LogAttributeType.CONTEXT); + logEventMap.put(logAttrKey, mdcEntry.getValue()); + } } if (level.toString().isEmpty()) { @@ -122,101 +106,9 @@ private static boolean shouldCreateLogEvent(boolean messageEmpty, Throwable thro return !messageEmpty || !ExceptionUtil.isThrowableNull(throwable); } - private static int getDefaultNumOfLogEventAttributes(Map mdcPropertyMap) { - return isApplicationLoggingForwardingIncludeMdcEnabled() + private static int calculateInitialMapSize(Map mdcPropertyMap) { + return AppLoggingUtils.isAppLoggingContextDataEnabled() ? mdcPropertyMap.size() + DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES : DEFAULT_NUM_OF_LOG_EVENT_ATTRIBUTES; } - - /** - * Gets a String representing the agent linking metadata in blob format: - * NR-LINKING|entity.guid|hostname|trace.id|span.id|entity.name| - * - * @return agent linking metadata string blob - */ - public static String getLinkingMetadataBlob() { - Map agentLinkingMetadata = NewRelic.getAgent().getLinkingMetadata(); - StringBuilder blob = new StringBuilder(); - blob.append(" ").append(BLOB_PREFIX).append(BLOB_DELIMITER); - - if (agentLinkingMetadata != null && agentLinkingMetadata.size() > 0) { - appendAttributeToBlob(agentLinkingMetadata.get(ENTITY_GUID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(HOSTNAME), blob); - appendAttributeToBlob(agentLinkingMetadata.get(TRACE_ID), blob); - appendAttributeToBlob(agentLinkingMetadata.get(SPAN_ID), blob); - appendAttributeToBlob(urlEncode(agentLinkingMetadata.get(ENTITY_NAME)), blob); - } - return blob.toString(); - } - - private static void appendAttributeToBlob(String attribute, StringBuilder blob) { - if (attribute != null && !attribute.isEmpty()) { - blob.append(attribute); - } - blob.append(BLOB_DELIMITER); - } - - /** - * URL encode a String value. - * - * @param value String to encode - * @return URL encoded String - */ - static String urlEncode(String value) { - try { - if (value != null) { - value = URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); - } - } catch (UnsupportedEncodingException e) { - NewRelic.getAgent() - .getLogger() - .log(java.util.logging.Level.WARNING, "Unable to URL encode entity.name for application_logging.local_decorating", e); - } - return value; - } - - /** - * Check if all application_logging features are enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.enabled", APP_LOGGING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging metrics feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingMetricsEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.metrics.enabled", APP_LOGGING_METRICS_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging forwarding feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingForwardingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.enabled", APP_LOGGING_FORWARDING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging local_decorating feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingLocalDecoratingEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.local_decorating.enabled", APP_LOGGING_LOCAL_DECORATING_DEFAULT_ENABLED); - } - - /** - * Check if the application_logging forwarding include_mdc feature is enabled. - * - * @return true if enabled, else false - */ - public static boolean isApplicationLoggingForwardingIncludeMdcEnabled() { - return NewRelic.getAgent().getConfig().getValue("application_logging.forwarding.include_mdc.enabled", APP_LOGGING_FORWARDING_INCLUDE_MDC_DEFAULT_ENABLED); - } } diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java deleted file mode 100644 index 31a627f9ad..0000000000 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/AgentUtilTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.nr.agent.instrumentation.logbackclassic12; - -import org.junit.Assert; -import org.junit.Test; - -public class AgentUtilTest { - - @Test - public void testUrlEncoding() { - final String ENCODED_PIPE = "%7C"; - final String ENCODED_SPACE = "+"; - // The main goal of the encoding is to eliminate | characters from the entity.name as | is used as - // the BLOB_DELIMITER for separating the agent metadata attributes that are appended to log files - final String valueToEncode = "|My Application|"; - final String expectedEncodedValue = ENCODED_PIPE + "My" + ENCODED_SPACE + "Application" + ENCODED_PIPE; - - String encodedValue = AgentUtil.urlEncode(valueToEncode); - - Assert.assertEquals(expectedEncodedValue, encodedValue); - } - -} diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/ContextDataTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/ContextDataTest.java new file mode 100644 index 0000000000..22518c276b --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/ContextDataTest.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package com.nr.agent.instrumentation.logbackclassic12; + +import ch.qos.logback.classic.Logger; +import com.newrelic.agent.introspec.InstrumentationTestConfig; +import com.newrelic.agent.introspec.InstrumentationTestRunner; +import com.newrelic.agent.introspec.Introspector; +import com.newrelic.agent.model.LogEvent; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import java.util.Collection; +import java.util.Map; + +import static org.junit.Assert.*; + +@RunWith(InstrumentationTestRunner.class) +@InstrumentationTestConfig(includePrefixes = { "ch.qos.logback" }, configName = "application_logging_context_data_enabled.yml") +public class ContextDataTest { + private final Introspector introspector = InstrumentationTestRunner.getIntrospector(); + + @Before + public void reset() { + introspector.clearLogEvents(); + } + + @Test + public void contextDataEnabledTest() { + + final Logger logger = (Logger) LoggerFactory.getLogger(Logger_InstrumentationTest.class); + String attrKey1 = "key"; + String attrKey2 = "anotherKey"; + String val1 = "42"; + String val2 = "panic"; + + MDC.put(attrKey1, val1); + MDC.put(attrKey2, val2); + logger.error("message"); + + Collection logEvents = introspector.getLogEvents(); + assertEquals(1, logEvents.size()); + + // verify included context attrs + LogEvent logEvent = logEvents.iterator().next(); + Map attributes = logEvent.getUserAttributesCopy(); + long contextAttrCount = attributes.keySet().stream() + .filter(key -> key.startsWith("context.")) + .count(); + // MDC data is filtered later in the process + assertEquals(2L, contextAttrCount); + assertEquals(val1, attributes.get("context." + attrKey1)); + assertEquals(val2, attributes.get("context." + attrKey2)); + } + +} diff --git a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java index 4586af70a7..fe6f51a962 100644 --- a/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java +++ b/instrumentation/logback-classic-1.2/src/test/java/com/nr/agent/instrumentation/logbackclassic12/LoggingEvent_InstrumentationTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import java.util.Collection; import java.util.List; @@ -360,4 +361,24 @@ private List getErrorLevelLogEvents(Collection logEvents) { .filter(logEvent -> logEvent.getUserAttributesCopy().containsValue(Level.ERROR)) .collect(Collectors.toList()); } + + @Test + public void contextDataDisabledTest() { + + final Logger logger = (Logger) LoggerFactory.getLogger(Logger_InstrumentationTest.class); + MDC.put("include", "42"); + MDC.put("common", "life, the universe and everything"); + MDC.put("exclude", "panic"); + logger.error("message"); + + Collection logEvents = introspector.getLogEvents(); + assertEquals(1, logEvents.size()); + + // verify no context attrs + LogEvent logEvent = logEvents.iterator().next(); + long contextAttrCount = logEvent.getUserAttributesCopy().keySet().stream() + .filter(key -> key.startsWith("context.")) + .count(); + assertEquals(0, contextAttrCount); + } } diff --git a/instrumentation/logback-classic-1.2/src/test/resources/application_logging_context_data_enabled.yml b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_context_data_enabled.yml new file mode 100644 index 0000000000..1212edc2e9 --- /dev/null +++ b/instrumentation/logback-classic-1.2/src/test/resources/application_logging_context_data_enabled.yml @@ -0,0 +1,12 @@ +common: &default_settings + application_logging: + enabled: true + forwarding: + enabled: true + max_samples_stored: 10000 + context_data: + enabled: true + metrics: + enabled: true + local_decorating: + enabled: false diff --git a/instrumentation/spring-aop-2/build.gradle b/instrumentation/spring-aop-2/build.gradle index 1acb66e45f..059bd44adb 100644 --- a/instrumentation/spring-aop-2/build.gradle +++ b/instrumentation/spring-aop-2/build.gradle @@ -4,9 +4,6 @@ dependencies { implementation("org.springframework:spring-aop:3.2.+") // implementation("org.springframework:spring:2.0.3") // First version to have MethodMatchers.matches() - testImplementation("org.powermock:powermock-module-junit4-rule-agent:1.6.1") - testImplementation("org.powermock:powermock-module-junit4:1.6.1") - testImplementation("org.powermock:powermock-api-mockito:1.6.1") } jar { diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java index 1c8f04b32f..78e35f9d72 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/DummyTransaction.java @@ -12,6 +12,7 @@ import com.newrelic.agent.bridge.TracedMethod; import com.newrelic.agent.bridge.WebResponse; import com.newrelic.agent.bridge.datastore.DatabaseVendor; +import com.newrelic.agent.bridge.logging.LogAttributeKey; import com.newrelic.agent.browser.BrowserTransactionState; import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.config.CrossProcessConfig; @@ -664,7 +665,7 @@ public void recordCustomEvent(String eventType, Map attributes) { static final class DummyLogs implements Logs { @Override - public void recordLogEvent(Map attributes) { + public void recordLogEvent(Map attributes) { } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributesNode.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributesNode.java index 9fece37cca..397e8c74a9 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributesNode.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/AttributesNode.java @@ -79,7 +79,7 @@ protected Boolean applyRules(String key) { } private void logMatch(String key) { - if (Agent.LOG.isFinerEnabled()) { + if (Agent.LOG.isFinestEnabled()) { Agent.LOG.log(Level.FINEST, "{0}: Attribute key \"{1}\" matched {2} {3} rule \"{4}\"", destination, key, isDefaultRule ? "default" : "config", includeDestination ? "INCLUDE" : "EXCLUDE", original); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/DisabledExcludeIncludeFilter.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/DisabledExcludeIncludeFilter.java new file mode 100644 index 0000000000..0ba7490104 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/DisabledExcludeIncludeFilter.java @@ -0,0 +1,25 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.attributes; + +/** + * Implementation of {@link ExcludeIncludeFilter} that always returns false. + */ +public class DisabledExcludeIncludeFilter implements ExcludeIncludeFilter { + + public static final ExcludeIncludeFilter INSTANCE = new DisabledExcludeIncludeFilter(); + + private DisabledExcludeIncludeFilter() { + // use the INSTANCE + } + + @Override + public boolean shouldInclude(String key) { + return false; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilter.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilter.java new file mode 100644 index 0000000000..8ddd92e7e0 --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilter.java @@ -0,0 +1,13 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.attributes; + + +public interface ExcludeIncludeFilter { + boolean shouldInclude(String key); +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImpl.java new file mode 100644 index 0000000000..b28acd5b3d --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImpl.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.attributes; + +import java.util.Collection; +import java.util.Collections; + +/** + *

A filter that tells if a given key should be included, given include and exclude lists.

+ *

Items in the list may have wildcards ('*') as the last character.

+ *

If the include list is empty, then all keys are included.

+ */ +public class ExcludeIncludeFilterImpl implements ExcludeIncludeFilter { + + private final RootConfigAttributesNode rootNode; + + private final boolean includeByDefault; + + /** + * @param identifier used in logs so filters can be distinguished + * @param excludes the collection of keys to be excluded + * @param includes the collection of keys to be included, if empty, all keys are considered included + */ + public ExcludeIncludeFilterImpl(String identifier, Collection excludes, Collection includes) { + rootNode = new RootConfigAttributesNode(identifier); + + this.includeByDefault = includes == null || includes.isEmpty(); + + boolean notDefault = false; + + if (includes != null) { + for (String key : includes) { + AttributesNode includeNode = new AttributesNode(key, true, identifier, notDefault); + rootNode.addNode(includeNode); + } + } + + if (excludes != null) { + for (String key : excludes) { + AttributesNode excludeNode = new AttributesNode(key, false, identifier, notDefault); + rootNode.addNode(excludeNode); + } + } + } + + @Override + public boolean shouldInclude(String key) { + Boolean apply = rootNode.applyRules(key); + // apply can be null, which means there was no match for the key. In this case, the default value will be used. + return apply == null ? includeByDefault : apply; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java index 924a3f59f2..87b1364419 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfig.java @@ -7,6 +7,8 @@ package com.newrelic.agent.config; +import java.util.List; + /** * Configuration for application logging features. These settings do not pertain to agent logs. */ @@ -33,6 +35,13 @@ public interface ApplicationLoggingConfig { */ boolean isForwardingEnabled(); + /** + * Allow the agent to forward context data along with the application logs to New Relic. + * + * @return true if context data forwarding is enabled, false otherwise + */ + boolean isForwardingContextDataEnabled(); + /** * Allow the agent to decorate application log files and console output with New Relic specific linking metadata. * @@ -47,4 +56,17 @@ public interface ApplicationLoggingConfig { */ int getMaxSamplesStored(); + /** + * Get the include list for context data. + * + * @return + */ + List getForwardingContextDataInclude(); + + /** + * Get the exclude list for context data. + * + * @return a list of Strings + */ + List getForwardingContextDataExclude(); } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java index 1ea07bc8e8..6c6e0c2ea3 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingConfigImpl.java @@ -8,6 +8,7 @@ package com.newrelic.agent.config; import java.util.Collections; +import java.util.List; import java.util.Map; import static com.newrelic.agent.config.AgentConfigImpl.APPLICATION_LOGGING; @@ -15,10 +16,15 @@ /* Default config should look like: * * application_logging: + * enabled: true * forwarding: * enabled: true * max_samples_stored: 10000 + * context_data: + * enabled: false + * include: + * exclude: * metrics: * enabled: true * local_decorating: @@ -93,4 +99,19 @@ public boolean isForwardingEnabled() { public int getMaxSamplesStored() { return applicationLoggingForwardingConfig.getMaxSamplesStored(); } + + @Override + public boolean isForwardingContextDataEnabled() { + return applicationLoggingEnabled && applicationLoggingForwardingConfig.isContextDataEnabled(); + } + + @Override + public List getForwardingContextDataInclude() { + return applicationLoggingForwardingConfig.contextDataInclude(); + } + + @Override + public List getForwardingContextDataExclude() { + return applicationLoggingForwardingConfig.contextDataExclude(); + } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfig.java new file mode 100644 index 0000000000..7459e27fdf --- /dev/null +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfig.java @@ -0,0 +1,56 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import com.newrelic.agent.Agent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; + +public class ApplicationLoggingContextDataConfig extends BaseConfig { + public static final String ROOT = "context_data"; + public static final String ENABLED = "enabled"; + public static final String INCLUDE = "include"; + public static final String EXCLUDE = "exclude"; + + public static final boolean DEFAULT_ENABLED = false; + + private final boolean enabled; + private final List include; + private final List exclude; + + + public ApplicationLoggingContextDataConfig(Map props, String parentRoot, boolean highSecurity) { + super(props, parentRoot + ROOT + "."); + enabled = !highSecurity && getProperty(ENABLED, DEFAULT_ENABLED); + + if (enabled) { + include = getUniqueStrings(INCLUDE); + exclude = getUniqueStrings(EXCLUDE); + } else { + include = Collections.emptyList(); + exclude = Collections.emptyList(); + } + } + + public boolean getEnabled() { + return enabled; + } + + public List getInclude() { + return include; + } + + public List getExclude() { + return exclude; + } +} diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java index 5d301d154b..a9bbe6837e 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfig.java @@ -7,8 +7,12 @@ package com.newrelic.agent.config; +import com.google.common.collect.Sets; import com.newrelic.agent.Agent; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -22,12 +26,14 @@ public class ApplicationLoggingForwardingConfig extends BaseConfig { private final boolean enabled; private final int maxSamplesStored; + private final ApplicationLoggingContextDataConfig contextDataConfig; public ApplicationLoggingForwardingConfig(Map props, String parentRoot, boolean highSecurity) { super(props, parentRoot + ROOT + "."); maxSamplesStored = initMaxSamplesStored(); boolean storedMoreThan0 = maxSamplesStored > 0; enabled = storedMoreThan0 && !highSecurity && getProperty(ENABLED, DEFAULT_ENABLED); + contextDataConfig = createContextDataConfig(highSecurity); } private int initMaxSamplesStored() { @@ -40,6 +46,11 @@ private int initMaxSamplesStored() { } } + private ApplicationLoggingContextDataConfig createContextDataConfig(boolean highSecurity) { + Map contextDataProps = getProperty(ApplicationLoggingContextDataConfig.ROOT, Collections.emptyMap()); + return new ApplicationLoggingContextDataConfig(contextDataProps, systemPropertyPrefix, highSecurity); + } + public boolean getEnabled() { return enabled; } @@ -47,4 +58,16 @@ public boolean getEnabled() { public int getMaxSamplesStored() { return maxSamplesStored; } + + public boolean isContextDataEnabled() { + return enabled && contextDataConfig.getEnabled(); + } + + public List contextDataInclude() { + return contextDataConfig.getInclude(); + } + + public List contextDataExclude() { + return contextDataConfig.getExclude(); + } } diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/analytics/InsightsServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/analytics/InsightsServiceImpl.java index 4029d7d331..134a9cf6ca 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/analytics/InsightsServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/analytics/InsightsServiceImpl.java @@ -377,8 +377,8 @@ private static CustomInsightsEvent createValidatedEvent(String eventType, Map entry : attributes.entrySet()) { + String key = entry.getKey(); Object value = entry.getValue(); // key or value is null, skip it with a log message and iterate to next entry in attributes.entrySet() diff --git a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java index 44feb0b67e..a1003af400 100644 --- a/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java +++ b/newrelic-agent/src/main/java/com/newrelic/agent/service/logging/LogSenderServiceImpl.java @@ -20,8 +20,16 @@ import com.newrelic.agent.TransactionData; import com.newrelic.agent.attributes.AttributeSender; import com.newrelic.agent.attributes.AttributeValidator; +import com.newrelic.agent.attributes.DisabledExcludeIncludeFilter; +import com.newrelic.agent.attributes.ExcludeIncludeFilter; +import com.newrelic.agent.attributes.ExcludeIncludeFilterImpl; +import com.newrelic.agent.bridge.logging.LogAttributeKey; +import com.newrelic.agent.bridge.logging.LogAttributeType; import com.newrelic.agent.config.AgentConfig; import com.newrelic.agent.config.AgentConfigListener; +import com.newrelic.agent.config.ApplicationLoggingConfig; +import com.newrelic.agent.config.ApplicationLoggingContextDataConfig; +import com.newrelic.agent.config.ApplicationLoggingForwardingConfig; import com.newrelic.agent.model.LogEvent; import com.newrelic.agent.service.AbstractService; import com.newrelic.agent.service.ServiceFactory; @@ -65,6 +73,8 @@ public class LogSenderServiceImpl extends AbstractService implements LogSenderSe public static final String METHOD = "add log event attribute"; public static final String LOG_SENDER_SERVICE = "Log Sender Service"; + private volatile ExcludeIncludeFilter contextDataKeyFilter; + /** * Lifecycle listener for log events associated with a transaction */ @@ -95,13 +105,17 @@ public void dispatcherTransactionCancelled(Transaction transaction) { protected final AgentConfigListener configListener = new AgentConfigListener() { @Override public void configChanged(String appName, AgentConfig agentConfig) { + ApplicationLoggingConfig appLoggingConfig = agentConfig.getApplicationLoggingConfig(); + // if the config has changed for the app, just remove it and regenerate enabled next transaction isEnabledForApp.remove(appName); - forwardingEnabled = agentConfig.getApplicationLoggingConfig().isForwardingEnabled(); - maxSamplesStored = agentConfig.getApplicationLoggingConfig().getMaxSamplesStored(); - boolean metricsEnabled = agentConfig.getApplicationLoggingConfig().isMetricsEnabled(); - boolean localDecoratingEnabled = agentConfig.getApplicationLoggingConfig().isLocalDecoratingEnabled(); + maxSamplesStored = appLoggingConfig.getMaxSamplesStored(); + forwardingEnabled = appLoggingConfig.isForwardingEnabled(); + contextDataKeyFilter = createContextDataKeyFilter(appLoggingConfig); + + boolean metricsEnabled = appLoggingConfig.isMetricsEnabled(); + boolean localDecoratingEnabled = appLoggingConfig.isLocalDecoratingEnabled(); recordApplicationLoggingSupportabilityMetrics(forwardingEnabled, metricsEnabled, localDecoratingEnabled); } }; @@ -133,13 +147,28 @@ public void recordApplicationLoggingSupportabilityMetrics(boolean forwardingEnab public LogSenderServiceImpl() { super(LogSenderServiceImpl.class.getSimpleName()); AgentConfig config = ServiceFactory.getConfigService().getDefaultAgentConfig(); - maxSamplesStored = config.getApplicationLoggingConfig().getMaxSamplesStored(); - forwardingEnabled = config.getApplicationLoggingConfig().isForwardingEnabled(); + ApplicationLoggingConfig appLoggingConfig = config.getApplicationLoggingConfig(); + + maxSamplesStored = appLoggingConfig.getMaxSamplesStored(); + forwardingEnabled = appLoggingConfig.isForwardingEnabled(); + contextDataKeyFilter = createContextDataKeyFilter(appLoggingConfig); + isEnabledForApp.put(config.getApplicationName(), forwardingEnabled); } + private ExcludeIncludeFilter createContextDataKeyFilter(ApplicationLoggingConfig appLoggingConfig) { + if (appLoggingConfig.isForwardingContextDataEnabled()) { + List include = appLoggingConfig.getForwardingContextDataInclude(); + List exclude = appLoggingConfig.getForwardingContextDataExclude(); + return new ExcludeIncludeFilterImpl("application_logging.forwarding.context_data", exclude, include); + } else { + return DisabledExcludeIncludeFilter.INSTANCE; + } + } + /** * Whether the LogSenderService is enabled or not + * * @return true if enabled, else false */ @Override @@ -180,12 +209,13 @@ private void removeHarvestables() { /** * Records a LogEvent. If a LogEvent occurs within a Transaction it will be associated with it. + * * @param attributes A map of log event data (e.g. log message, log timestamp, log level) * Each key should be a String and each value should be a String, Number, or Boolean. * For map values that are not String, Number, or Boolean object types the toString value will be used. */ @Override - public void recordLogEvent(Map attributes) { + public void recordLogEvent(Map attributes) { if (logEventsDisabled() || attributes == null || attributes.isEmpty()) { return; } @@ -300,16 +330,16 @@ public void storeEvent(String appName, LogEvent event) { /** * Create and store a LogEvent instance - * @param appName application name + * + * @param appName application name * @param attributes Map of attributes to create a LogEvent from */ - private void createAndStoreEvent(String appName, Map attributes) { + private void createAndStoreEvent(String appName, Map attributes) { if (logEventsDisabled()) { return; } - DistributedSamplingPriorityQueue eventList = getReservoir(appName); - eventList.add(createValidatedEvent(attributes)); + eventList.add(createValidatedEvent(attributes, contextDataKeyFilter)); Agent.LOG.finest(MessageFormat.format("Added event of type {0}", LOG_EVENT_TYPE)); } @@ -469,10 +499,12 @@ private static String mapInternString(String value) { /** * Create a validated LogEvent - * @param attributes Map of attributes to create a LogEvent from + * + * @param attributes Map of attributes to create a LogEvent from + * @param contextDataKeyFilter * @return LogEvent instance */ - private static LogEvent createValidatedEvent(Map attributes) { + private static LogEvent createValidatedEvent(Map attributes, ExcludeIncludeFilter contextDataKeyFilter) { Map logEventLinkingMetadata = AgentLinkingMetadata.getLogEventLinkingMetadata(TraceMetadataImpl.INSTANCE, ServiceFactory.getConfigService(), ServiceFactory.getRPMService()); // Initialize new logEventAttributes map with agent linking metadata @@ -487,31 +519,34 @@ private static LogEvent createValidatedEvent(Map attributes) { // within the attribute sender, the modified value won't be "interned" in our map. AttributeSender sender = new LogEventAttributeSender(logEventAttributes); - for (Map.Entry entry : attributes.entrySet()) { - String key = entry.getKey(); + for (Map.Entry entry : attributes.entrySet()) { + LogAttributeKey logAttrKey = entry.getKey(); Object value = entry.getValue(); // key or value is null, skip it with a log message and iterate to next entry in attributes.entrySet() - if (key == null || value == null) { + if (logAttrKey == null || logAttrKey.getKey() == null || value == null) { Agent.LOG.log(Level.WARNING, "Log event with invalid attributes key or value of null was reported for a transaction but ignored." + " Each key should be a String and each value should be a String, Number, or Boolean."); continue; } - mapInternString(key); + // filter out context attrs that should not be included + if (logAttrKey.type == LogAttributeType.CONTEXT && !contextDataKeyFilter.shouldInclude(logAttrKey.getKey())) { + continue; + } + String prefixedKey = mapInternString(logAttrKey.getPrefixedKey()); if (value instanceof String) { - sender.addAttribute(key, mapInternString((String) value), METHOD); + sender.addAttribute(prefixedKey, mapInternString((String) value), METHOD); } else if (value instanceof Number) { - sender.addAttribute(key, (Number) value, METHOD); + sender.addAttribute(prefixedKey, (Number) value, METHOD); } else if (value instanceof Boolean) { - sender.addAttribute(key, (Boolean) value, METHOD); + sender.addAttribute(prefixedKey, (Boolean) value, METHOD); } else { // Java Agent specific - toString the value. This allows for e.g. enums as arguments. - sender.addAttribute(key, mapInternString(value.toString()), METHOD); + sender.addAttribute(prefixedKey, mapInternString(value.toString()), METHOD); } } - return event; } @@ -546,28 +581,30 @@ protected Map getAttributeMap() { @Override public Logs getTransactionLogs(AgentConfig config) { - return new TransactionLogs(config); + return new TransactionLogs(config, contextDataKeyFilter); } /** * Used to record LogEvents on Transactions */ public static final class TransactionLogs implements Logs { - final LinkedBlockingQueue events; + private final LinkedBlockingQueue events; + private final ExcludeIncludeFilter contextDataKeyFilter; - TransactionLogs(AgentConfig config) { + TransactionLogs(AgentConfig config, ExcludeIncludeFilter contextDataKeyFilter) { int maxSamplesStored = config.getApplicationLoggingConfig().getMaxSamplesStored(); events = new LinkedBlockingQueue<>(maxSamplesStored); + this.contextDataKeyFilter = contextDataKeyFilter; } @Override - public void recordLogEvent(Map attributes) { + public void recordLogEvent(Map attributes) { if (ServiceFactory.getConfigService().getDefaultAgentConfig().isHighSecurity()) { Agent.LOG.log(Level.FINER, "Event of type {0} not collected due to high security mode being enabled.", LOG_EVENT_TYPE); return; } - LogEvent event = createValidatedEvent(attributes); + LogEvent event = createValidatedEvent(attributes, contextDataKeyFilter); if (events.offer(event)) { Agent.LOG.log(Level.FINEST, "Added event of type {0} in Transaction.", LOG_EVENT_TYPE); } else { diff --git a/newrelic-agent/src/main/resources/newrelic.yml b/newrelic-agent/src/main/resources/newrelic.yml index 8b22b65094..9e2fabac96 100644 --- a/newrelic-agent/src/main/resources/newrelic.yml +++ b/newrelic-agent/src/main/resources/newrelic.yml @@ -105,12 +105,18 @@ common: &default_settings # Default is 10000. Setting to 0 will disable. #max_samples_stored: 10000 - # Whether the log events should include MDC from loggers with support for that. - include_mdc: + # Whether the log events should include context from loggers with support for that. + include_context_data: - # When true, application logs will contain MDC data. + # When true, application logs will contain context data. enabled: false + # A comma separated list of attribute keys whose values should be sent to New Relic. + #include: + + # A comma separated list of attribute keys whose values should not be sent to New Relic. + #exclude: + # The agent will generate metrics to indicate the number of # application log events occurring at each distinct log level. metrics: diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImplTest.java new file mode 100644 index 0000000000..31424e503d --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/attributes/ExcludeIncludeFilterImplTest.java @@ -0,0 +1,34 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.attributes; + +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.*; + +public class ExcludeIncludeFilterImplTest { + + @Test + public void testExplicitInclude() { + ExcludeIncludeFilterImpl filter = new ExcludeIncludeFilterImpl("any", Collections.singleton("exclude"), Collections.singleton("include")); + assertFalse(filter.shouldInclude("exclude")); + assertFalse(filter.shouldInclude("asdf")); + assertTrue(filter.shouldInclude("include")); + } + + @Test + public void testIncludeByDefault() { + ExcludeIncludeFilterImpl filter = new ExcludeIncludeFilterImpl("any", Collections.singleton("exclude"), Collections.emptySet()); + assertFalse(filter.shouldInclude("exclude")); + assertTrue(filter.shouldInclude("asdf")); + assertTrue(filter.shouldInclude("include")); + } + +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfigTest.java new file mode 100644 index 0000000000..32c5a9693d --- /dev/null +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingContextDataConfigTest.java @@ -0,0 +1,112 @@ +/* + * + * * Copyright 2022 New Relic Corporation. All rights reserved. + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.newrelic.agent.config; + +import com.newrelic.agent.SaveSystemPropertyProviderRule; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.*; + +public class ApplicationLoggingContextDataConfigTest { + + @Rule + public SaveSystemPropertyProviderRule saveSystemPropertyProviderRule = new SaveSystemPropertyProviderRule(); + + @Test + public void defaultForwardingConfig() { + Map localProps = new HashMap<>(); + boolean highSecurityDisabled = false; + + ApplicationLoggingContextDataConfig config = new ApplicationLoggingContextDataConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + highSecurityDisabled); + + // assert default context data config is created + assertFalse(config.getEnabled()); + assertTrue(config.getInclude().isEmpty()); + assertTrue(config.getExclude().isEmpty()); + } + + @Test + public void testEnable() { + Map localProps = new HashMap<>(); + localProps.put("enabled", Boolean.TRUE); + localProps.put("include", "asdf,qwer"); + boolean highSecurityDisabled = false; + + ApplicationLoggingContextDataConfig config = new ApplicationLoggingContextDataConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + highSecurityDisabled); + + assertTrue(config.getEnabled()); + assertEquals(2, config.getInclude().size()); + assertThat(config.getInclude(), contains("asdf", "qwer")); + assertTrue(config.getExclude().isEmpty()); + } + + @Test + public void highSecurityModeOverrides() { + Map localProps = new HashMap<>(); + localProps.put("enabled", Boolean.TRUE); + localProps.put("include", "asdf,qwer"); + localProps.put("exclude", "1234,zxcv"); + boolean highSecurityEnabled = true; + + ApplicationLoggingContextDataConfig config = new ApplicationLoggingContextDataConfig(localProps, ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT, + highSecurityEnabled); + + assertFalse(config.getEnabled()); + assertTrue(config.getInclude().isEmpty()); + assertTrue(config.getExclude().isEmpty()); + } + + @Test + public void usesEnvVarForNestedConfig() { + Map envVars = new HashMap<>(); + envVars.put("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_ENABLED", "true"); + envVars.put("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_INCLUDE", "include"); + envVars.put("NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_EXCLUDE", "exclude"); + + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade(envVars) + )); + + ApplicationLoggingContextDataConfig config = new ApplicationLoggingContextDataConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT + "forwarding.", false); + + assertTrue(config.getEnabled()); + assertThat(config.getInclude(), contains("include")); + assertThat(config.getExclude(), contains("exclude")); + } + + @Test + public void usesSysPropForNestedConfig() { + Properties properties = new Properties(); + properties.put("newrelic.config.application_logging.forwarding.context_data.enabled", "true"); + properties.put("newrelic.config.application_logging.forwarding.context_data.include", "include"); + properties.put("newrelic.config.application_logging.forwarding.context_data.exclude", "exclude"); + SystemPropertyFactory.setSystemPropertyProvider(new SystemPropertyProvider( + new SaveSystemPropertyProviderRule.TestSystemProps(properties), + new SaveSystemPropertyProviderRule.TestEnvironmentFacade() + )); + + ApplicationLoggingContextDataConfig config = new ApplicationLoggingContextDataConfig(Collections.emptyMap(), + ApplicationLoggingConfigImpl.SYSTEM_PROPERTY_ROOT + "forwarding.", false); + + assertTrue(config.getEnabled()); + assertThat(config.getInclude(), contains("include")); + assertThat(config.getExclude(), contains("exclude")); + } + +} \ No newline at end of file diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java index 16cfe5802c..18d0fa27e1 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/config/ApplicationLoggingForwardingConfigTest.java @@ -12,6 +12,7 @@ import java.util.Properties; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ApplicationLoggingForwardingConfigTest { diff --git a/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java index 430c79417e..460b9057d6 100644 --- a/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java +++ b/newrelic-agent/src/test/java/com/newrelic/agent/service/logging/LogSenderServiceImplTest.java @@ -8,6 +8,10 @@ import com.newrelic.agent.Transaction; import com.newrelic.agent.TransactionData; import com.newrelic.agent.TransactionService; +import com.newrelic.agent.attributes.ExcludeIncludeFilter; +import com.newrelic.agent.attributes.ExcludeIncludeFilterImpl; +import com.newrelic.agent.bridge.logging.LogAttributeKey; +import com.newrelic.agent.bridge.logging.LogAttributeType; import com.newrelic.agent.config.AgentConfigImpl; import com.newrelic.agent.config.ApplicationLoggingConfigImpl; import com.newrelic.agent.config.ApplicationLoggingForwardingConfig; @@ -85,14 +89,14 @@ public void testHighSecurity() throws Exception { when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( - AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + AgentConfigImpl.createAgentConfig(Collections.emptyMap()), allowAllFilter()); when(transaction.getLogEventData()).thenReturn(logs); when(transaction.getApplicationName()).thenReturn(appName); when(transaction.isInProgress()).thenReturn(true); - logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); - logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); - logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + logSenderService.recordLogEvent(createAgentLogAttrs("field", "value")); + logSenderService.recordLogEvent(createAgentLogAttrs("field2", "value2")); + logSenderService.recordLogEvent(createAgentLogAttrs("field3", "value3")); MockRPMService analyticsData = new MockRPMService(); when(ServiceFactory.getServiceManager().getRPMServiceManager().getRPMService(appName)).thenReturn( @@ -106,7 +110,7 @@ public void testHighSecurity() throws Exception { logSenderService.harvestHarvestables(); assertEquals(0, analyticsData.getEvents().size()); - assertEquals(0, logs.events.size()); + assertEquals(0, logs.getEventsForTesting().size()); } @Test @@ -117,9 +121,9 @@ public void testNoTransaction() throws Exception { verify(txService, times(1)).addTransactionListener(logSenderService.transactionListener); - logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); - logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); - logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + logSenderService.recordLogEvent(createAgentLogAttrs("field", "value")); + logSenderService.recordLogEvent(createAgentLogAttrs("field2", "value2")); + logSenderService.recordLogEvent(createAgentLogAttrs("field3", "value3")); MockRPMService analyticsData = new MockRPMService(); when(ServiceFactory.getServiceManager().getRPMServiceManager().getOrCreateRPMService(appName)).thenReturn( @@ -141,14 +145,14 @@ public void testWithTransaction() throws Exception { when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( - AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + AgentConfigImpl.createAgentConfig(Collections.emptyMap()), allowAllFilter()); when(transaction.getLogEventData()).thenReturn(logs); when(transaction.getApplicationName()).thenReturn(appName); when(transaction.isInProgress()).thenReturn(true); - logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); - logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); - logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + logSenderService.recordLogEvent(createAgentLogAttrs("field", "value")); + logSenderService.recordLogEvent(createAgentLogAttrs("field2", "value2")); + logSenderService.recordLogEvent(createAgentLogAttrs("field3", "value3")); MockRPMService analyticsData = new MockRPMService(); when(ServiceFactory.getServiceManager().getRPMServiceManager().getOrCreateRPMService(appName)).thenReturn( @@ -157,7 +161,7 @@ public void testWithTransaction() throws Exception { logSenderService.harvestHarvestables(); assertEquals(0, analyticsData.getEvents().size()); - assertEquals(3, logs.events.size()); + assertEquals(3, logs.getEventsForTesting().size()); } @Test @@ -169,14 +173,14 @@ public void testTransactionHarvest() throws Exception { when(ServiceFactory.getTransactionService().getTransaction(false)).thenReturn(transaction); LogSenderServiceImpl.TransactionLogs logs = new LogSenderServiceImpl.TransactionLogs( - AgentConfigImpl.createAgentConfig(Collections.emptyMap())); + AgentConfigImpl.createAgentConfig(Collections.emptyMap()), allowAllFilter()); when(transaction.getLogEventData()).thenReturn(logs); when(transaction.getApplicationName()).thenReturn(appName); when(transaction.isInProgress()).thenReturn(true); - logSenderService.recordLogEvent(ImmutableMap.of("field", "value")); - logSenderService.recordLogEvent(ImmutableMap.of("field2", "value2")); - logSenderService.recordLogEvent(ImmutableMap.of("field3", "value3")); + logSenderService.recordLogEvent(createAgentLogAttrs("field", "value")); + logSenderService.recordLogEvent(createAgentLogAttrs("field2", "value2")); + logSenderService.recordLogEvent(createAgentLogAttrs("field3", "value3")); // these should be filtered out logSenderService.recordLogEvent(null); @@ -234,4 +238,13 @@ private static Map createConfig(Boolean highSecurity, Integer as config.put(AgentConfigImpl.APP_NAME, appName); return config; } + + private static Map createAgentLogAttrs(String key, String value) { + LogAttributeKey logKey = new LogAttributeKey(key, LogAttributeType.AGENT); + return ImmutableMap.of(logKey, value); + } + + private ExcludeIncludeFilter allowAllFilter() { + return new ExcludeIncludeFilterImpl("allowAll", Collections.emptySet(), Collections.emptySet()); + } } \ No newline at end of file diff --git a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml index 1d5467afd6..ecf426ef77 100644 --- a/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml +++ b/newrelic-agent/src/test/resources/com/newrelic/agent/config/newrelic.yml @@ -22,7 +22,7 @@ common: &default_settings enabled: true forwarding: enabled: true - include_mdc: + include_context_data: enabled: false max_samples_stored: 10000 metrics: