Merge branch '7.x' into backport/7.x/pr-80705

elastic · Oct 27, 2020 · e3524c2 · e3524c2
2 parents 47c8bed + dd49b7e
commit e3524c2
Show file tree

Hide file tree

Showing 330 changed files with 1,422 additions and 691 deletions.
diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts
@@ -28,6 +28,8 @@ import {
 
 import { ApmConfiguration } from './config';
 
+const initialEnv = { ...process.env };
+
 describe('ApmConfiguration', () => {
   beforeEach(() => {
     packageMock.raw = {
@@ -39,6 +41,7 @@ describe('ApmConfiguration', () => {
   });
 
   afterEach(() => {
+    process.env = { ...initialEnv };
     resetAllMocks();
   });
 
@@ -90,13 +93,16 @@ describe('ApmConfiguration', () => {
     let config = new ApmConfiguration(mockedRootDir, {}, false);
     expect(config.getConfig('serviceName')).toEqual(
       expect.objectContaining({
-        serverUrl: expect.any(String),
-        secretToken: expect.any(String),
+        breakdownMetrics: true,
       })
     );
 
     config = new ApmConfiguration(mockedRootDir, {}, true);
-    expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl');
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        breakdownMetrics: false,
+      })
+    );
   });
 
   it('loads the configuration from the kibana config file', () => {
@@ -156,4 +162,32 @@ describe('ApmConfiguration', () => {
       })
     );
   });
+
+  it('correctly sets environment', () => {
+    delete process.env.ELASTIC_APM_ENVIRONMENT;
+    delete process.env.NODE_ENV;
+
+    let config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'development',
+      })
+    );
+
+    process.env.NODE_ENV = 'production';
+    config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'production',
+      })
+    );
+
+    process.env.ELASTIC_APM_ENVIRONMENT = 'ci';
+    config = new ApmConfiguration(mockedRootDir, {}, false);
+    expect(config.getConfig('serviceName')).toEqual(
+      expect.objectContaining({
+        environment: 'ci',
+      })
+    );
+  });
 });
diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts
@@ -26,32 +26,26 @@ import { readFileSync } from 'fs';
 import { ApmAgentConfig } from './types';
 
 const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => {
-  if (isDistributable) {
-    return {
-      active: false,
-      globalLabels: {},
-      // Do not use a centralized controlled config
-      centralConfig: false,
-      // Capture all exceptions that are not caught
-      logUncaughtExceptions: true,
-      // Can be performance intensive, disabling by default
-      breakdownMetrics: false,
-    };
-  }
-
+  // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html
   return {
-    active: false,
-    serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443',
+    active: process.env.ELASTIC_APM_ACTIVE || false,
+    environment: process.env.ELASTIC_APM_ENVIRONMENT || process.env.NODE_ENV || 'development',
+
+    serverUrl: 'https://b1e3b4b4233e44cdad468c127d0af8d8.apm.europe-west1.gcp.cloud.es.io:443',
+
     // The secretToken below is intended to be hardcoded in this file even though
     // it makes it public. This is not a security/privacy issue. Normally we'd
     // instead disable the need for a secretToken in the APM Server config where
     // the data is transmitted to, but due to how it's being hosted, it's easier,
     // for now, to simply leave it in.
-    secretToken: 'R0Gjg46pE9K9wGestd',
+    secretToken: '2OyjjaI6RVkzx2O5CV',
+
+    logUncaughtExceptions: true,
     globalLabels: {},
-    breakdownMetrics: true,
     centralConfig: false,
-    logUncaughtExceptions: true,
+
+    // Can be performance intensive, disabling by default
+    breakdownMetrics: isDistributable ? false : true,
   };
 };
 
@@ -84,7 +78,8 @@ export class ApmConfiguration {
         getDefaultConfig(this.isDistributable),
         this.getConfigFromKibanaConfig(),
         this.getDevConfig(),
-        this.getDistConfig()
+        this.getDistConfig(),
+        this.getCIConfig()
       );
 
       const rev = this.getGitRev();
@@ -146,6 +141,21 @@ export class ApmConfiguration {
     };
   }
 
+  private getCIConfig(): ApmAgentConfig {
+    if (process.env.ELASTIC_APM_ENVIRONMENT !== 'ci') {
+      return {};
+    }
+
+    return {
+      globalLabels: {
+        branch: process.env.ghprbSourceBranch || '',
+        targetBranch: process.env.ghprbTargetBranch || '',
+        ciJobName: process.env.JOB_NAME || '',
+        ciBuildNumber: process.env.BUILD_NUMBER || '',
+      },
+    };
+  }
+
   private getGitRev() {
     if (this.isDistributable) {
       return this.pkgBuild.sha;

diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts
@@ -146,7 +146,11 @@ run(
     await lastValueFrom(update$.pipe(logOptimizerState(log, config)));
 
     if (updateLimits) {
-      updateBundleLimits(log, config);
+      updateBundleLimits({
+        log,
+        config,
+        dropMissing: !(focus || filter),
+      });
     }
   },
   {

diff --git a/packages/kbn-optimizer/src/limits.ts b/packages/kbn-optimizer/src/limits.ts
@@ -79,10 +79,18 @@ export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerCo
   log.success('limits.yml file valid');
 }
 
-export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) {
+interface UpdateBundleLimitsOptions {
+  log: ToolingLog;
+  config: OptimizerConfig;
+  dropMissing: boolean;
+}
+
+export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) {
   const metrics = getMetrics(log, config);
 
-  const pageLoadAssetSize: NonNullable<Limits['pageLoadAssetSize']> = {};
+  const pageLoadAssetSize: NonNullable<Limits['pageLoadAssetSize']> = dropMissing
+    ? {}
+    : config.limits.pageLoadAssetSize ?? {};
 
   for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) {
     if (metric.group === 'page load bundle size') {

diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh
@@ -24,6 +24,10 @@ export NODE_OPTIONS="$NODE_OPTIONS --max-old-space-size=4096"
 ###
 export FORCE_COLOR=1
 
+### APM tracking
+###
+export ELASTIC_APM_ENVIRONMENT=ci
+
 ###
 ### check that we seem to be in a kibana project
 ###

diff --git a/...ution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json b/...ution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json
@@ -20,9 +20,9 @@
   "rule_id": "a87a4e42-1d82-44bd-b0bf-d9b7f91fb89e",
   "severity": "medium",
   "tags": [
-    "APM",
-    "Elastic"
+    "Elastic",
+    "APM"
   ],
   "type": "query",
-  "version": 3
+  "version": 4
 }
diff --git a/...ver/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json b/...ver/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json
@@ -20,9 +20,9 @@
   "rule_id": "75ee75d8-c180-481c-ba88-ee50129a6aef",
   "severity": "medium",
   "tags": [
-    "APM",
-    "Elastic"
+    "Elastic",
+    "APM"
   ],
   "type": "query",
-  "version": 3
+  "version": 4
 }
diff --git a/...ity_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json b/...ity_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json
@@ -38,9 +38,9 @@
   "rule_id": "43303fd4-4839-4e48-b2b2-803ab060758d",
   "severity": "medium",
   "tags": [
-    "APM",
-    "Elastic"
+    "Elastic",
+    "APM"
   ],
   "type": "query",
-  "version": 3
+  "version": 4
 }
diff --git a/...y_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json b/...y_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json
@@ -20,9 +20,9 @@
   "rule_id": "d49cc73f-7a16-4def-89ce-9fc7127d7820",
   "severity": "medium",
   "tags": [
-    "APM",
-    "Elastic"
+    "Elastic",
+    "APM"
   ],
   "type": "query",
-  "version": 3
+  "version": 4
 }
diff --git a/...b/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json b/...b/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json
diff --git a/...r/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json b/...r/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json
@@ -25,11 +25,12 @@
   "rule_id": "594e0cbf-86cc-45aa-9ff7-ff27db27d3ed",
   "severity": "low",
   "tags": [
-    "AWS",
     "Elastic",
+    "Cloud",
+    "AWS",
+    "Continuous Monitoring",
     "SecOps",
-    "Logging",
-    "Continuous Monitoring"
+    "Log Auditing"
   ],
   "threat": [
     {

diff --git a/...etection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json b/...etection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json
@@ -22,10 +22,11 @@
   "severity": "low",
   "tags": [
     "Elastic",
+    "Cloud",
     "GCP",
     "Continuous Monitoring",
     "SecOps",
-    "Logging"
+    "Log Auditing"
   ],
   "threat": [
     {

diff --git a/...r/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json b/...r/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json
@@ -22,10 +22,11 @@
   "severity": "low",
   "tags": [
     "Elastic",
+    "Cloud",
     "GCP",
     "Continuous Monitoring",
     "SecOps",
-    "Logging"
+    "Log Auditing"
   ],
   "threat": [
     {

diff --git a/...r/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json b/...r/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json
@@ -22,11 +22,12 @@
   "rule_id": "b6dce542-2b75-4ffb-b7d6-38787298ba9d",
   "severity": "medium",
   "tags": [
-    "Azure",
     "Elastic",
-    "SecOps",
+    "Cloud",
+    "Azure",
     "Continuous Monitoring",
-    "Logging"
+    "SecOps",
+    "Log Auditing"
   ],
   "threat": [
     {

diff --git a/...ction_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json b/...ction_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json
@@ -17,7 +17,10 @@
   "severity": "low",
   "tags": [
     "Elastic",
-    "Windows"
+    "Host",
+    "Windows",
+    "Threat Detection",
+    "Command and Control"
   ],
   "threat": [
     {

diff --git a/...ib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json b/...ib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json
@@ -23,7 +23,9 @@
   "severity": "high",
   "tags": [
     "Elastic",
-    "Network"
+    "Network",
+    "Threat Detection",
+    "Command and Control"
   ],
   "threat": [
     {

diff --git a/...tion_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json b/...tion_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json
@@ -23,7 +23,9 @@
   "severity": "medium",
   "tags": [
     "Elastic",
-    "Network"
+    "Network",
+    "Threat Detection",
+    "Command and Control"
   ],
   "threat": [
     {
@@ -43,5 +45,5 @@
     }
   ],
   "type": "query",
-  "version": 4
+  "version": 5
 }
diff --git a/...ne/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json b/...ne/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json
@@ -23,7 +23,9 @@
   "severity": "medium",
   "tags": [
     "Elastic",
-    "Network"
+    "Network",
+    "Threat Detection",
+    "Command and Control"
   ],
   "threat": [
     {