-
Notifications
You must be signed in to change notification settings - Fork 143
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1542 from newrelic/stall-detection-duplicate
Stall detection duplicate
- Loading branch information
Showing
10 changed files
with
663 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 25 additions & 0 deletions
25
newrelic-agent/src/main/java/com/newrelic/agent/config/SlowTransactionsConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| /* | ||
| * | ||
| * * Copyright 2020 New Relic Corporation. All rights reserved. | ||
| * * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package com.newrelic.agent.config; | ||
|
|
||
| import com.newrelic.agent.service.slowtransactions.SlowTransactionService; | ||
|
|
||
| public interface SlowTransactionsConfig { | ||
|
|
||
| /** | ||
| * True if the {@link SlowTransactionService} is enabled, else false. | ||
| */ | ||
| boolean isEnabled(); | ||
|
|
||
| /** | ||
| * The minimum number of milliseconds a transaction must be running to be | ||
| * reported as slow. | ||
| */ | ||
| long getThresholdMillis(); | ||
|
|
||
| } |
41 changes: 41 additions & 0 deletions
41
newrelic-agent/src/main/java/com/newrelic/agent/config/SlowTransactionsConfigImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| /* | ||
| * | ||
| * * Copyright 2020 New Relic Corporation. All rights reserved. | ||
| * * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package com.newrelic.agent.config; | ||
|
|
||
| import java.util.Map; | ||
|
|
||
| public class SlowTransactionsConfigImpl extends BaseConfig implements SlowTransactionsConfig { | ||
|
|
||
| public static final String ROOT = "slow_transactions"; | ||
| public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config." + ROOT + "."; | ||
| public static final String ENABLED = "enabled"; | ||
| public static final String THRESHOLD = "threshold"; | ||
|
|
||
| public static final boolean DEFAULT_ENABLED = false; | ||
| public static final int DEFAULT_THRESHOLD_MILLIS = 1000; | ||
|
|
||
| private final boolean isEnabled; | ||
| private final int thresholdMillis; | ||
|
|
||
| public SlowTransactionsConfigImpl(Map<String, Object> pProps) { | ||
| super(pProps, SYSTEM_PROPERTY_ROOT); | ||
| isEnabled = getProperty(ENABLED, DEFAULT_ENABLED); | ||
| thresholdMillis = getIntProperty(THRESHOLD, DEFAULT_THRESHOLD_MILLIS); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isEnabled() { | ||
| return isEnabled; | ||
| } | ||
|
|
||
| @Override | ||
| public long getThresholdMillis() { | ||
| return thresholdMillis; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
214 changes: 214 additions & 0 deletions
214
...ent/src/main/java/com/newrelic/agent/service/slowtransactions/SlowTransactionService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,214 @@ | ||
| package com.newrelic.agent.service.slowtransactions; | ||
|
|
||
| import com.newrelic.agent.ExtendedTransactionListener; | ||
| import com.newrelic.agent.HarvestListener; | ||
| import com.newrelic.agent.MetricNames; | ||
| import com.newrelic.agent.Transaction; | ||
| import com.newrelic.agent.TransactionData; | ||
| import com.newrelic.agent.config.AgentConfig; | ||
| import com.newrelic.agent.config.SlowTransactionsConfig; | ||
| import com.newrelic.agent.model.CustomInsightsEvent; | ||
| import com.newrelic.agent.service.AbstractService; | ||
| import com.newrelic.agent.service.ServiceFactory; | ||
| import com.newrelic.agent.service.analytics.InsightsService; | ||
| import com.newrelic.agent.stats.StatsEngine; | ||
| import com.newrelic.agent.stats.TransactionStats; | ||
| import com.newrelic.agent.tracing.DistributedTraceServiceImpl; | ||
| import com.newrelic.agent.util.StackTraces; | ||
| import com.newrelic.api.agent.NewRelic; | ||
|
|
||
| import javax.annotation.Nullable; | ||
| import java.lang.management.ManagementFactory; | ||
| import java.lang.management.ThreadInfo; | ||
| import java.lang.management.ThreadMXBean; | ||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.logging.Level; | ||
|
|
||
| public class SlowTransactionService extends AbstractService implements ExtendedTransactionListener, HarvestListener { | ||
|
|
||
| private final ConcurrentHashMap<String, Transaction> openTransactions = new ConcurrentHashMap<>(); | ||
| private final ThreadMXBean threadMXBean; | ||
|
|
||
| private final boolean isEnabled; | ||
| private final long thresholdMillis; | ||
| private final int maxStackTraceLines; | ||
|
|
||
| @Nullable | ||
| private InsightsService insightsService; | ||
|
|
||
| public SlowTransactionService(AgentConfig agentConfig) { | ||
| this(agentConfig, ManagementFactory.getThreadMXBean()); | ||
| } | ||
|
|
||
| // Visible for testing | ||
| SlowTransactionService(AgentConfig agentConfig, ThreadMXBean threadMXBean) { | ||
| super(SlowTransactionService.class.getSimpleName()); | ||
| SlowTransactionsConfig slowTransactionsConfig = agentConfig.getSlowTransactionsConfig(); | ||
| this.isEnabled = slowTransactionsConfig.isEnabled(); | ||
| this.thresholdMillis = slowTransactionsConfig.getThresholdMillis(); | ||
| this.maxStackTraceLines = agentConfig.getMaxStackTraceLines(); | ||
| this.threadMXBean = threadMXBean; | ||
|
|
||
| NewRelic.getAgent().getMetricAggregator().incrementCounter( | ||
| agentConfig.getSlowTransactionsConfig().isEnabled() ? | ||
| MetricNames.SUPPORTABILITY_SLOW_TXN_DETECTION_ENABLED : MetricNames.SUPPORTABILITY_SLOW_TXN_DETECTION_DISABLED); | ||
| } | ||
|
|
||
| @Override | ||
| protected void doStart() throws Exception { | ||
| // Short circuit if disabled | ||
| if (!isEnabled) { | ||
| return; | ||
| } | ||
| ServiceFactory.getTransactionService().addTransactionListener(this); | ||
| ServiceFactory.getHarvestService().addHarvestListener(this); | ||
| insightsService = ServiceFactory.getServiceManager().getInsights(); | ||
| } | ||
|
|
||
| @Override | ||
| protected void doStop() throws Exception { | ||
| // Short circuit if disabled | ||
| if (!isEnabled) { | ||
| return; | ||
| } | ||
| ServiceFactory.getTransactionService().removeTransactionListener(this); | ||
| ServiceFactory.getHarvestService().removeHarvestListener(this); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isEnabled() { | ||
| return isEnabled; | ||
| } | ||
|
|
||
| @Override | ||
| public void dispatcherTransactionStarted(Transaction transaction) { | ||
| if (getLogger().isLoggable(Level.FINEST)) { | ||
| getLogger().finest("Transaction started with id " + transaction.getGuid()); | ||
| } | ||
| openTransactions.put(transaction.getGuid(), transaction); | ||
| } | ||
|
|
||
| @Override | ||
| public void dispatcherTransactionCancelled(Transaction transaction) { | ||
| if (getLogger().isLoggable(Level.FINEST)) { | ||
| getLogger().finest("Transaction cancelled with guid " + transaction.getGuid()); | ||
| } | ||
| openTransactions.remove(transaction.getGuid()); | ||
| } | ||
|
|
||
| @Override | ||
| public void dispatcherTransactionFinished(TransactionData transactionData, TransactionStats transactionStats) { | ||
| if (getLogger().isLoggable(Level.FINEST)) { | ||
| getLogger().finest("Transaction finished with guid " + transactionData.getGuid()); | ||
| } | ||
| openTransactions.remove(transactionData.getGuid()); | ||
| } | ||
|
|
||
| // Visible for testing | ||
| Map<String, Transaction> getOpenTransactions() { | ||
| return Collections.unmodifiableMap(openTransactions); | ||
| } | ||
|
|
||
| @Override | ||
| public void beforeHarvest(String appName, StatsEngine statsEngine) { | ||
| run(); | ||
| } | ||
|
|
||
| @Override | ||
| public void afterHarvest(String appName) { | ||
| } | ||
|
|
||
| // Visible for testing | ||
| void run() { | ||
| if (getLogger().isLoggable(Level.FINE)) { | ||
| getLogger().fine("Identifying slow threads. Open transactions: " + openTransactions.size()); | ||
| } | ||
| Transaction slowestOpen = null; | ||
| long slowestOpenMillis = thresholdMillis; | ||
|
|
||
| // Identify the slowest open transaction we haven't yet reported | ||
| for (Transaction transaction : openTransactions.values()) { | ||
| long openMs = System.currentTimeMillis() - transaction.getWallClockStartTimeMs(); | ||
| if (openMs > slowestOpenMillis) { | ||
| slowestOpen = transaction; | ||
| slowestOpenMillis = openMs; | ||
| } | ||
| } | ||
|
|
||
| if (slowestOpen == null) { | ||
| getLogger().fine("No new slow transactions identified."); | ||
| return; | ||
| } | ||
|
|
||
| // Construct and record SlowTransaction event | ||
| Map<String, Object> attributes = extractMetadata(slowestOpen, slowestOpenMillis); | ||
| String guid = slowestOpen.getGuid(); | ||
| if (getLogger().isLoggable(Level.FINE)) { | ||
| getLogger().fine("Slowest open transaction has guid " | ||
| + guid + " has been open for " + slowestOpenMillis + "ms, attributes: " + attributes); | ||
| } | ||
| if (insightsService != null) { | ||
| logger.fine("Sending slow transaction"); | ||
| insightsService.storeEvent( | ||
| ServiceFactory.getRPMService().getApplicationName(), | ||
| new CustomInsightsEvent( | ||
| "SlowTransaction", | ||
| System.currentTimeMillis(), | ||
| attributes, | ||
| DistributedTraceServiceImpl.nextTruncatedFloat())); | ||
| //insightsService.recordCustomEvent("SlowTransaction", attributes); | ||
| } | ||
| // Remove from openTransactions to ensure we don't report the same Transaction | ||
| // multiple times | ||
| openTransactions.remove(guid); | ||
| } | ||
|
|
||
| // Visible for testing | ||
| Map<String, Object> extractMetadata(Transaction transaction, long openMillis) { | ||
| Map<String, Object> attributes = new HashMap<>(); | ||
|
|
||
| // Attributes | ||
| // Write attributes first so hardcoded attributes are prioritized | ||
| attributes.putAll(transaction.getUserAttributes()); | ||
| attributes.putAll(transaction.getErrorAttributes()); | ||
| attributes.putAll(transaction.getAgentAttributes()); | ||
| attributes.putAll(transaction.getIntrinsicAttributes()); | ||
|
|
||
| // General | ||
| attributes.put("guid", transaction.getGuid()); | ||
| attributes.put("name", transaction.getPriorityTransactionName().getName()); | ||
| attributes.put("transactionType", transaction.getPriorityTransactionName().getCategory()); | ||
| attributes.put("timestamp", transaction.getWallClockStartTimeMs()); | ||
| attributes.put("elapsed_ms", openMillis); | ||
|
|
||
| // Initiating thread info | ||
| long initiatingThreadId = transaction.getInitiatingThreadId(); | ||
| attributes.put("thread.id", initiatingThreadId); | ||
| ThreadInfo threadInfo = threadMXBean.getThreadInfo(initiatingThreadId, maxStackTraceLines); | ||
| if (threadInfo != null) { | ||
| attributes.put("thread.name", threadInfo.getThreadName()); | ||
| attributes.put("thread.state", threadInfo.getThreadState().name()); | ||
| List<StackTraceElement> scrubbedStackTraceElements = StackTraces.scrubAndTruncate(threadInfo.getStackTrace()); | ||
| attributes.put("code.stacktrace", stackTraceString(scrubbedStackTraceElements)); | ||
| } | ||
|
|
||
| new IllegalArgumentException().printStackTrace(); | ||
| return attributes; | ||
| } | ||
|
|
||
| private static String stackTraceString(List<StackTraceElement> stackTrace) { | ||
| if (stackTrace.isEmpty()) { | ||
| return ""; | ||
| } | ||
| StringBuilder stringBuilder = new StringBuilder(); | ||
| stringBuilder.append(stackTrace.get(0)).append("\n"); | ||
| for (int i = 1; i < stackTrace.size(); i++) { | ||
| stringBuilder.append("\tat ").append(stackTrace.get(i)).append("\n"); | ||
| } | ||
| return stringBuilder.toString(); | ||
| } | ||
| } |
Oops, something went wrong.