Merge remote-tracking branch 'upstream/master' into feature/use-arch-…

…in-the-package-binareis

* upstream/master:
  allows disable pod events enrichment with deployment name (elastic#28521)
  Remove Docker input from Filebeat (elastic#28817)
  [breaking] Make default_field: false the default for all fields (elastic#28596)
  Osquerybeat: Improve osquery client connect code (elastic#28848)
  Add crawler  metrics into the stats metricset for Enterprise Search (elastic#28790)
  Remove the now deprecated appsearch module from metricbeat (elastic#28850)
  Remove Beat generators (elastic#28816)
  chore: upload files to Google Storage when they exist (elastic#28836)
  Revert "chore(ci): disable E2E tests in Beats (elastic#28715)" (elastic#28812)
  Deprecate generating custom Beats (elastic#28814)
  [Metricbeat] upgrade flatbuffers to 1.12.1 (elastic#28094)
  Osquerybeat: Fix restart flags after previously bad config (elastic#28827)
  Force ECS and JSON logging for libbeat/logp (elastic#28573)
  Filebeat: Error on startup for unconfigured module (elastic#28818)
  Deprecate log input in favour of filestream (elastic#28623)
  Fix some spelling mistakes (elastic#28080)

.ci/packaging.groovy

@@ -246,8 +246,7 @@ pipeline {
           agent { label 'ubuntu-18 && immutable' }
           options { skipDefaultCheckout() }
           steps {
-            log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
-            //runE2ETests()
+            runE2ETests()
           }
         }
       }

CHANGELOG-developer.next.asciidoc

@@ -56,6 +56,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 - Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
+- Removed Beat generators. {pull}28816[28816]
+- libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 
 ==== Bugfixes
 
@@ -129,3 +131,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 ==== Deprecated
 
 - Deprecated the `common.Float` type. {issue}28279[28279] {pull}28280[28280]
+- Deprecate Beat generators. {pull}28814[28814]

CHANGELOG.next.asciidoc

@@ -20,11 +20,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Libbeat: logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 - Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
 - Update docker client. {pull}28716[28716]
 - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]
 - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
+- Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
 
 *Auditbeat*
 
@@ -58,6 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fail to start Filebat if none between `queue_url`, `bucket_arn` or `non_aws_bucket_name` is set for a configured aws-s3 input {issue}13911[13911] {pull}28666[28666]
 - All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620]
 - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620]
+- Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817]
 
 *Heartbeat*
 
@@ -137,6 +140,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
 - Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
 - Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395]
+- Allows disable pod events enrichment with deployment name {pull}28521[28521]
 - Fix AWS proxy_url config from url to string type.  {pull}28725[28725]
 - Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]
 
@@ -378,6 +382,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Deprecate `log` input in favour of `filestream` input. {pull}28623[28623]
 
 *Heartbeat*
 

Jenkinsfile

@@ -617,8 +617,7 @@ def target(Map args = [:]) {
           pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
         }
         if(isE2E) {
-          log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
-          //e2e(args)
+          e2e(args)
         }
       }
     }
@@ -830,20 +829,23 @@ def archiveTestOutput(Map args = [:]) {
       catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
         withMageEnv(version: "${env.GO_VERSION}"){
           dir(directory){
-            cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+            cmd(label: "Archive system tests files", script: 'mage packageSystemTests', returnStatus: true)
           }
         }
+
         def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
         def files = findFiles(glob: "${fileName}")
-        files.each { file ->
-          echo "${file.name}"
+
+        if (files?.length() > 0) {
+          googleStorageUploadExt(
+            bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+            credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+            pattern: "${fileName}",
+            sharedPublicly: true
+          )
+        } else {
+          log(level: 'WARN', text: "There are no system-tests files to upload Google Storage}")
         }
-        googleStorageUploadExt(
-          bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
-          credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
-          pattern: "${fileName}",
-          sharedPublicly: true
-        )
       }
     }
   }

Jenkinsfile.yml

@@ -2,8 +2,6 @@ projects:
     - "auditbeat"
     - "deploy/kubernetes"
     - "filebeat"
-    # Skipping because they are failing, see https://github.com/elastic/beats/pull/28723
-    #- "generator"
     - "heartbeat"
     - "libbeat"
     - "metricbeat"

NOTICE.txt

@@ -10108,11 +10108,11 @@ Contents of probable licence file $GOMODCACHE/github.com/gomodule/redigo@v1.8.3/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/google/flatbuffers
-Version: v1.12.0
+Version: v1.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.12.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.12.1/LICENSE.txt:
 
 
                                  Apache License

auditbeat/auditbeat.reference.yml

@@ -1448,14 +1448,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/module/file_integrity/eventreader_test.go

@@ -41,7 +41,7 @@ func init() {
 const ErrorSharingViolation syscall.Errno = 32
 
 func TestEventReader(t *testing.T) {
-	t.Skip("Flaky test: about 1/10 of bulds fails https://github.com/elastic/beats/issues/21302")
+	t.Skip("Flaky test: about 1/10 of builds fails https://github.com/elastic/beats/issues/21302")
 	// Make dir to monitor.
 	dir, err := ioutil.TempDir("", "audit")
 	if err != nil {
@@ -241,7 +241,7 @@ func TestEventReader(t *testing.T) {
 }
 
 func TestRaces(t *testing.T) {
-	t.Skip("Flaky test: about 1/20 of bulds fails https://github.com/elastic/beats/issues/21303")
+	t.Skip("Flaky test: about 1/20 of builds fails https://github.com/elastic/beats/issues/21303")
 	const (
 		fileMode os.FileMode = 0640
 		N                    = 100

auditbeat/module/file_integrity/fileinfo_windows.go

@@ -94,7 +94,7 @@ func fileOwner(path string) (sid, owner string, err error) {
 	}
 	defer syscall.LocalFree((syscall.Handle)(unsafe.Pointer(securityDescriptor)))
 
-	// Covert SID to a string and lookup the username.
+	// Convert SID to a string and lookup the username.
 	var errs multierror.Errors
 	sid, err = securityID.String()
 	if err != nil {

auditbeat/tests/system/test_file_integrity.py

@@ -98,7 +98,7 @@ def test_non_recursive(self):
 
             # wait until file1 is reported before deleting. Otherwise the hash
             # might not be calculated
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file1)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file1)), ignore_case=True)
 
             os.unlink(file1)
 
@@ -107,8 +107,9 @@ def test_non_recursive(self):
             file3 = os.path.join(subdir, "other_file.txt")
             self.create_file(file3, "not reported.")
 
-            self.wait_log_contains("\"deleted\"")
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(subdir)), ignore_case=True)
+            # log entries are JSON formatted, this value shows up as an escaped json string.
+            self.wait_log_contains("\\\"deleted\\\"")
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True)
             self.wait_output(3)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output()))
@@ -157,7 +158,7 @@ def test_recursive(self):
             # wait until the directories to watch are printed in the logs
             # this happens when the file_integrity module starts
             self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
-            self.wait_log_contains("\"recursive\": true")
+            self.wait_log_contains("\"recursive\":true")
 
             # auditbeat_test/subdir/
             subdir = os.path.join(dirs[0], "subdir")
@@ -173,7 +174,7 @@ def test_recursive(self):
             file2 = os.path.join(subdir2, "more.txt")
             self.create_file(file2, "")
 
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file2)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True)
             self.wait_output(4)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output()))

dev-tools/ecs-migration.yml

@@ -54,7 +54,7 @@
   alias6: true
   alias: true
 
-- from: docker.container.labels # TODO: How to map these?
+- from: docker.container.labels  # TODO: How to map these?
   to: container.labels
   alias6: false
   alias: true
@@ -115,8 +115,8 @@
 
 - from: source
   to:
-    - log.file.path
-    - log.source.address
+  - log.file.path
+  - log.source.address
   alias: false
   beat: filebeat
 
@@ -428,7 +428,7 @@
   beat: filebeat
 
 - from: suricata.eve.timestamp
-  to: "@timestamp"
+  to: '@timestamp'
   alias: true
   beat: filebeat
 
@@ -476,7 +476,7 @@
   beat: filebeat
 
 - from: system.auth.timestamp
-  to: "@timestamp"
+  to: '@timestamp'
   alias: true
   beat: filebeat
 
@@ -1599,6 +1599,7 @@
   alias: true
   beat: metricbeat
 
+
 ### Redis
 
 - from: php_fpm.status.pid
@@ -1872,7 +1873,7 @@
 - from: method
   to: http.request.method
   alias: false
-  comment: Field is used by serveral protocols.
+  comment: Field is used by several protocols.
   beat: packetbeat
 
 - from: path
@@ -1883,15 +1884,15 @@
 - from: real_ip
   to: network.forwarded_ip
   alias: false
-  comment: Field is used by serveral protocols.
+  comment: Field is used by several protocols.
   beat: packetbeat
 
 ## MySQL
 - from: mysql.iserror
   to: status
   alias: false
   comment: >
-    Status reflects whether or not an error occured. Its values are either
+    Status reflects whether or not an error occurred. Its values are either
     OK or Error.
   beat: packetbeat
 

dev-tools/jenkins_release.sh

@@ -45,7 +45,7 @@ cleanup() {
 trap cleanup EXIT
 
 # This controls the defaults used the Jenkins package job. They can be
-# overridden by setting them in the environement prior to running this script.
+# overridden by setting them in the environment prior to running this script.
 export SNAPSHOT="${SNAPSHOT:-true}"
 export PLATFORMS="${PLATFORMS:-+linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64}"
 

dev-tools/mage/fmt.go

@@ -50,13 +50,10 @@ func Format() {
 	mg.Deps(GoImports, PythonAutopep8)
 }
 
-// GoImports executes goimports against all .go files in and below the CWD. It
-// ignores vendor/ and generator/_templates/ directories.
+// GoImports executes goimports against all .go files in and below the CWD.
 func GoImports() error {
 	goFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
-		return filepath.Ext(path) == ".go" &&
-			!strings.Contains(path, "vendor/") &&
-			!strings.Contains(path, "generator/_templates/")
+		return filepath.Ext(path) == ".go"
 	})
 	if err != nil {
 		return err
@@ -84,9 +81,7 @@ func GoImports() error {
 // ignores build/ directories.
 func PythonAutopep8() error {
 	pyFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
-		return filepath.Ext(path) == ".py" &&
-			!strings.Contains(path, "build/") &&
-			!strings.Contains(path, "vendor/")
+		return filepath.Ext(path) == ".py" && !strings.Contains(path, "build/")
 	})
 	if err != nil {
 		return err

dev-tools/mage/gotest_test.go

@@ -139,7 +139,7 @@ func TestGoTest_CaptureOutput(t *testing.T) {
 
 			output := buf.String()
 			if !re.MatchString(output) {
-				t.Fatalf("GoTest output missmatch:\nwant:\n%v\n\ngot:\n%v", test.want, output)
+				t.Fatalf("GoTest output mismatch:\nwant:\n%v\n\ngot:\n%v", test.want, output)
 			}
 		})
 	}

dev-tools/mage/integtest.go

@@ -135,7 +135,7 @@ type IntegrationTester interface {
 	Use(dir string) (bool, error)
 	// HasRequirements returns an error if requirements are missing.
 	HasRequirements() error
-	// Test performs excecuting the test inside the environment.
+	// Test performs executing the test inside the environment.
 	Test(dir string, mageTarget string, env map[string]string) error
 	// InsideTest performs the actual test on the inside of environment.
 	InsideTest(test func() error) error

dev-tools/mage/kubernetes/kubectl.go

@@ -28,7 +28,7 @@ import (
 	"github.com/magefile/mage/sh"
 )
 
-// KubectlApply applys the manifest file to the kubernetes cluster.
+// KubectlApply applies the manifest file to the kubernetes cluster.
 //
 // KUBECONFIG must be in `env` to target a specific cluster.
 func KubectlApply(env map[string]string, stdout, stderr io.Writer, filepath string) error {
@@ -60,7 +60,7 @@ func KubectlDelete(env map[string]string, stdout, stderr io.Writer, filepath str
 	return err
 }
 
-// KubectlApplyInput applys the manifest string to the kubernetes cluster.
+// KubectlApplyInput applies the manifest string to the kubernetes cluster.
 //
 // KUBECONFIG must be in `env` to target a specific cluster.
 func KubectlApplyInput(env map[string]string, stdout, stderr io.Writer, manifest string) error {

docs/devguide/contributing.asciidoc

@@ -35,17 +35,6 @@ In the pull request, describe what your changes do and mention
 any bugs/issues related to the pull request. Please also add a changelog entry to
 https://github.com/elastic/beats/blob/master/CHANGELOG.next.asciidoc[CHANGELOG.next.asciidoc].
 
-[float]
-[[adding-new-beat]]
-=== Adding a New Beat
-
-If you want to create a new Beat, please read <<new-beat>>. You don't need to
-submit the code to this repository. Most new Beats start in their own repository
-and just make use of the libbeat packages. After you have a working Beat that
-you'd like to share with others, open a PR to add it to our list of
-https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.asciidoc[community
-Beats].
-
 [float]
 [[setting-up-dev-environment]]
 === Setting Up Your Dev Environment

docs/devguide/create-metricset.asciidoc

@@ -2,8 +2,7 @@
 === Creating a Metricset
 
 A metricset is the part of a Metricbeat module that fetches and structures the
-data from the remote service. Each module can have multiple metricsets. In this guide, you learn how to create your own metricset. If you want to create
-your own Beat that uses Metricbeat as a library, see <<creating-beat-from-metricbeat>>.
+data from the remote service. Each module can have multiple metricsets. In this guide, you learn how to create your own metricset.
 
 When creating a metricset for the first time, it generally helps to look at the
 implementation of existing metricsets for inspiration.