Obfuscate sql for external

.github/workflows/GHA-Functional-Tests.yaml

@@ -38,7 +38,6 @@ jobs:
         with:
           distribution: 'adopt'
           java-version: 8
-          cache: gradle
 
       - name: Save JAVA_HOME as JDK8 for later usage
         run: |
@@ -79,6 +78,9 @@ jobs:
           rm gradle.properties
           mv gradle.properties.gha gradle.properties
 
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       - name: Build newrelicJar
         if: ${{ env.GRADLE_KEY != '' }}
         env:

.github/workflows/GHA-Scala-Functional-Tests.yaml

@@ -38,7 +38,6 @@ jobs:
         with:
           distribution: 'adopt'
           java-version: 8
-          cache: gradle
 
       - name: Save JAVA_HOME as JDK8 for later usage
         run: |
@@ -67,6 +66,9 @@ jobs:
           rm gradle.properties
           mv gradle.properties.gha gradle.properties
 
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       - name: Build newrelicJar
         if: ${{ env.GRADLE_KEY != '' }}
         env:

.github/workflows/GHA-Scala-Instrumentation-Tests.yaml

@@ -45,7 +45,6 @@ jobs:
         with:
           distribution: 'adopt'
           java-version: 8
-          cache: gradle
 
       - name: Save JAVA_HOME as JDK8 for later usage
         run: |
@@ -90,6 +89,9 @@ jobs:
         run: find instrumentation -name "*.jar"
       ## End AWS jars - plan to cache (check for cache, restore if required)
 
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       - name: Build newrelicJar
         if: ${{ env.GRADLE_KEY != '' }}        
         env:

.github/workflows/GHA-Unit-Tests.yaml

@@ -39,7 +39,6 @@ jobs:
         with:
           distribution: 'adopt'
           java-version: 8
-          cache: 'gradle'
 
       - name: Save JAVA_HOME as JDK8 for later usage
         run: |
@@ -80,6 +79,9 @@ jobs:
           rm gradle.properties
           mv gradle.properties.gha gradle.properties
 
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       - name: Run unit tests for each Java version as defined in the matrix
         if: ${{ env.GRADLE_KEY != '' }}
         env:

.github/workflows/X-Reusable-Test.yml

@@ -41,7 +41,6 @@ jobs:
         with:
           distribution: 'adopt'
           java-version: 8
-          cache: 'gradle'
 
       # Save new JDK variable
       - name: Save JAVA_HOME as JDK8 for later usage
@@ -101,6 +100,9 @@ jobs:
         run: find instrumentation -name "*.jar"
       ## End AWS jars - plan to cache (check for cache, restore if required)
 
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       # Is newrelicJar present in newrelic-agent/build
       # TO DO: Below version number has to be dynamic
       - name: Build newrelicJar

functional_test/src/test/java/test/newrelic/test/agent/api/ApiTest.java

@@ -2039,7 +2039,6 @@ public String toObfuscatedQueryString(BsonDocument query) {
         Map<String, Object> attributes = rootTracer.getAgentAttributes();
 
         Assert.assertNotNull(attributes.get("sql"));
-        Assert.assertNotNull(attributes.get("sql_obfuscated"));
     }
 
     /* Messaging - FIT to Public API */

newrelic-agent/src/main/java/com/newrelic/agent/service/analytics/SpanEventFactory.java

@@ -353,9 +353,9 @@ private <T> String determineObfuscationLevel(SlowQueryDatastoreParameters<T> slo
         if (config.isHighSecurity() || config.getTransactionTracerConfig().getRecordSql().equals(SqlObfuscator.OFF_SETTING)) {
             return null;
         } else if (config.getTransactionTracerConfig().getRecordSql().equals(SqlObfuscator.RAW_SETTING)) {
-            return slowQueryDatastoreParameters.getQueryConverter().toRawQueryString(slowQueryDatastoreParameters.getRawQuery());
+            return slowQueryDatastoreParameters.getQueryConverter().toRawQueryString(slowQueryDatastoreParameters.getQuery());
         } else {
-            return slowQueryDatastoreParameters.getQueryConverter().toObfuscatedQueryString(slowQueryDatastoreParameters.getRawQuery());
+            return slowQueryDatastoreParameters.getQueryConverter().toObfuscatedQueryString(slowQueryDatastoreParameters.getQuery());
         }
     }
 

newrelic-agent/src/main/java/com/newrelic/agent/sql/DefaultSlowQueryListener.java

@@ -43,33 +43,22 @@ public DefaultSlowQueryListener(String appName, double thresholdInMillis) {
     @Override
     public <T> void noticeTracer(Tracer tracer, SlowQueryDatastoreParameters<T> slowQueryDatastoreParameters) {
         if (tracer.getDurationInMilliseconds() > thresholdInMillis) {
-            T rawQuery = slowQueryDatastoreParameters.getRawQuery();
-            QueryConverter<T> queryConverter = slowQueryDatastoreParameters.getQueryConverter();
-            if (rawQuery == null || queryConverter == null) {
+            String query = slowQueryDatastoreParameters.getQuery().toString();
+            if (query == null ) {
                 // Ignore tracer
                 return;
             }
 
-            String rawQueryString = queryConverter.toRawQueryString(rawQuery);
-            if (rawQueryString == null || rawQueryString.trim().isEmpty()) {
-                // Ignore tracer
-                return;
-            }
-
-            String obfuscatedQueryString = queryConverter.toObfuscatedQueryString(rawQuery);
-            if (obfuscatedQueryString == null) {
-                // Ignore tracer if no obfuscated query is provided
-                return;
-            }
-
             // Handle an "input query" from an ORM or a framework that automatically generates queries
             if (slowQueryDatastoreParameters instanceof SlowQueryWithInputDatastoreParameters) {
                 handleInputQuery(tracer, (SlowQueryWithInputDatastoreParameters) slowQueryDatastoreParameters);
             }
 
+            String recordSql = ServiceFactory.getConfigService().getDefaultAgentConfig().getTransactionTracerConfig().getRecordSql();
+            String sqlAttr = SqlObfuscator.RAW_SETTING.equals(recordSql) ?
+                    SqlTracer.SQL_PARAMETER_NAME : SqlTracer.SQL_OBFUSCATED_PARAMETER_NAME;
             // This allows transaction traces to show slow queries directly in the trace details
-            tracer.setAgentAttribute(SqlTracer.SQL_PARAMETER_NAME, rawQueryString);
-            tracer.setAgentAttribute(SqlTracer.SQL_OBFUSCATED_PARAMETER_NAME, obfuscatedQueryString);
+            tracer.setAgentAttribute(sqlAttr, query);
 
             DatastoreConfig datastoreConfig = ServiceFactory.getConfigService().getDefaultAgentConfig().getDatastoreConfig();
             boolean allUnknown = slowQueryDatastoreParameters.getHost() == null
@@ -89,16 +78,16 @@ public <T> void noticeTracer(Tracer tracer, SlowQueryDatastoreParameters<T> slow
                 slowQueryInfoCache = new BoundedConcurrentCache<>(MAX_SQL_TRACERS);
             }
 
-            SlowQueryInfo existingInfo = slowQueryInfoCache.get(obfuscatedQueryString);
+            SlowQueryInfo existingInfo = slowQueryInfoCache.get(query);
             if (existingInfo != null) {
                 // Aggregate tracers by SQL.
                 existingInfo.aggregate(tracer);
-                slowQueryInfoCache.putReplace(obfuscatedQueryString, existingInfo);
+                slowQueryInfoCache.putReplace(query, existingInfo);
             } else {
-                SlowQueryInfo sqlInfo = new SlowQueryInfo(null, tracer, rawQueryString, obfuscatedQueryString,
+                SlowQueryInfo sqlInfo = new SlowQueryInfo(null, tracer, query, query,
                         tracer.getTransactionActivity().getTransaction().getAgentConfig().getSqlTraceConfig());
                 sqlInfo.aggregate(tracer);
-                slowQueryInfoCache.putIfAbsent(obfuscatedQueryString, sqlInfo);
+                slowQueryInfoCache.putIfAbsent(query, sqlInfo);
             }
         }
     }

newrelic-agent/src/main/java/com/newrelic/agent/tracers/DefaultSqlTracer.java

@@ -201,7 +201,6 @@ protected void doFinish(Throwable throwable) {
                     String appName = getTransaction().getApplicationName();
                     SqlQueryConverter converter = new SqlQueryConverter(appName, getDatabaseVendor());
                     String obfuscatedQueryString = converter.toObfuscatedQueryString(sql.toString());
-
                     // Store the obfuscated query string
                     getTransaction().getIntrinsicAttributes().put(SQL_PARAMETER_NAME, obfuscatedQueryString);
                 }
@@ -264,13 +263,17 @@ public boolean isMetricProducer() {
     @Override
     protected void recordMetrics(TransactionStats transactionStats) {
         if (isMetricProducer() && getTransaction() != null) {
-            String rawSql = null;
+            String slowQuerySql = null;
             Object sqlObject = getSql();
+            String appName = getTransaction().getApplicationName();
+
             if (sqlObject != null) {
-                rawSql = new PreparedStatementSql(sql, params).toString();
+                slowQuerySql = new PreparedStatementSql(sql, params).toString();
+                if (queryExceedsSlowQueryThreshold(appName)) {
+                    slowQuerySql = getNoRawOrObfuscatedSql(slowQuerySql, appName);
+                }
             }
 
-            String appName = getTransaction().getApplicationName();
             String hostToReport = DatastoreMetrics.replaceLocalhost(getHost());
 
             if (getIdentifier() != null) {
@@ -280,7 +283,7 @@ protected void recordMetrics(TransactionStats transactionStats) {
                         .operation(parsedDatabaseStatement.getOperation())
                         .instance(hostToReport, getIdentifier())
                         .databaseName(getDatabaseName())
-                        .slowQuery(rawSql, new SqlQueryConverter(appName, getDatabaseVendor()))
+                        .slowQuery(slowQuerySql, new SqlQueryConverter(appName, getDatabaseVendor()))
                         .build());
             } else {
                 String portToReport = DatastoreMetrics.replacePort(getPort());
@@ -290,7 +293,7 @@ protected void recordMetrics(TransactionStats transactionStats) {
                         .operation(parsedDatabaseStatement.getOperation())
                         .instance(hostToReport, portToReport)
                         .databaseName(getDatabaseName())
-                        .slowQuery(rawSql, new SqlQueryConverter(appName, getDatabaseVendor()))
+                        .slowQuery(slowQuerySql, new SqlQueryConverter(appName, getDatabaseVendor()))
                         .build());
             }
 
@@ -301,6 +304,20 @@ protected void recordMetrics(TransactionStats transactionStats) {
         super.recordMetrics(transactionStats);
     }
 
+    /**
+     * This method is named this way because {@link SqlQueryConverter#toObfuscatedQueryString(String)} has
+     * unexpected conditional logic.
+     */
+    private String getNoRawOrObfuscatedSql(String rawSql, String appName) {
+        SqlQueryConverter converter = new SqlQueryConverter(appName, getDatabaseVendor());
+        return converter.toObfuscatedQueryString(rawSql);
+    }
+
+    private boolean queryExceedsSlowQueryThreshold(String appName) {
+        double threshold = ServiceFactory.getConfigService().getAgentConfig(appName).getTransactionTracerConfig().getExplainThresholdInMillis();
+        return getDurationInMilliseconds() > threshold;
+    }
+
     /**
      * Returns the sql for the statement instrumented by this driver. This method is overridden by prepared statement
      * tracers which return a sql object that can delay the rendering of the full sql statement with parameters until it
@@ -499,7 +516,7 @@ public String toString() {
      * @param parameters the parameter map
      * @return the parameterized SQL
      */
-    public static String parameterizeSql(String sql, Object[] parameters) throws Exception {
+    public static String parameterizeSql(String sql, Object[] parameters) {
         if (sql == null || parameters == null || parameters.length == 0) {
             return sql;
         }
@@ -512,11 +529,11 @@ public static String parameterizeSql(String sql, Object[] parameters) throws Exc
             } else {
                 Object val = i < parameters.length ? parameters[i] : null;
                 if (val instanceof Number) {
-                    sb.append(piece).append(val.toString());
+                    sb.append(piece).append(val);
                 } else if (val == null) {
                     sb.append(piece).append("?");
                 } else {
-                    sb.append(piece).append("'").append(val.toString()).append("'");
+                    sb.append(piece).append("'").append(val).append("'");
                 }
             }
         }
@@ -537,6 +554,12 @@ public String toRawQueryString(String rawQuery) {
             return rawQuery;
         }
 
+        /**
+         * For this implementation, the getSqlfuscator has conditional
+         * logic to return an obfuscator that respects the agent configuration setting `record_sql`
+         *
+         * See {@link com.newrelic.agent.database.DatabaseService#createSqlObfuscator(TransactionTracerConfig) }
+         */
         @Override
         public String toObfuscatedQueryString(String rawQuery) {
             SqlObfuscator sqlObfuscator = ServiceFactory.getDatabaseService().getSqlObfuscator(appName);

newrelic-agent/src/main/java/com/newrelic/agent/tracers/DefaultTracer.java

@@ -819,7 +819,7 @@ private void recordMessageBrokerMetrics(MessageConsumeParameters messageConsumeP
 
     private <T> void recordSlowQueryData(SlowQueryDatastoreParameters<T> slowQueryDatastoreParameters) {
         Transaction transaction = getTransactionActivity().getTransaction();
-        if (transaction != null && slowQueryDatastoreParameters.getRawQuery() != null
+        if (transaction != null && slowQueryDatastoreParameters.getQuery() != null
                 && slowQueryDatastoreParameters.getQueryConverter() != null) {
             // Attempt to record the slow query if it's above the threshold
             transaction.getSlowQueryListener(true).noticeTracer(this, slowQueryDatastoreParameters);

newrelic-api/src/main/java/com/newrelic/api/agent/SlowQueryDatastoreParameters.java

@@ -16,7 +16,7 @@
 public class SlowQueryDatastoreParameters<T> extends DatastoreParameters {
 
     /**
-     * The raw query object used for transforming into a raw query String and obfuscated query String
+     * The query object used for transforming into a raw query String and obfuscated query String
      */
     private final T rawQuery;
 
@@ -39,12 +39,12 @@ protected SlowQueryDatastoreParameters(SlowQueryDatastoreParameters<T> slowQuery
     }
 
     /**
-     * Returns the raw query object used for processing.
+     * Returns the query object used for processing. This query might be raw with parameters or obfuscated.
      *
-     * @return raw query object
+     * @return query object
      * @since 3.36.0
      */
-    public T getRawQuery() {
+    public T getQuery() {
         return rawQuery;
     }