Update beats framework to 64e6a72

This update includes packaging refactor (elastic/beats#7388)

NOTICE.txt

@@ -292,7 +292,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: c941a3d38792137bfc53a3a121f40cebf20b5512
+Revision: 64e6a72f19e37b2a46155ff293835e2494424655
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/.go-version

@@ -1 +1 @@
-1.10.2
+1.10.3

_beats/dev-tools/cmd/asset/asset.go

@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 // +build ignore
 
 package main
@@ -9,7 +26,6 @@ import (
 	"go/format"
 	"io/ioutil"
 	"os"
-	"text/template"
 
 	"github.com/elastic/beats/libbeat/asset"
 )
@@ -20,34 +36,6 @@ func init() {
 	pkg = flag.String("pkg", "", "Package name")
 }
 
-var tpl = template.Must(template.New("normalizations").Parse(`
-// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT.
-
-package {{ .Package }}
-
-import (
-	"github.com/elastic/beats/libbeat/asset"
-)
-
-func init() {
-	if err := asset.SetFields("{{ .Name }}", Asset); err != nil {
-		panic(err)
-	}
-}
-
-// Asset returns asset data
-func Asset() string {
-	return "{{ .Data }}"
-}
-
-`))
-
-type assetData struct {
-	Name    string
-	Data    string
-	Package string
-}
-
 func main() {
 	flag.Parse()
 
@@ -73,8 +61,9 @@ func main() {
 	}
 
 	var buf bytes.Buffer
-	tpl.Execute(&buf, assetData{
-		Name:    beatName + "/" + file,
+	asset.Template.Execute(&buf, asset.Data{
+		Beat:    beatName,
+		Name:    file,
 		Data:    encData,
 		Package: *pkg,
 	})

_beats/dev-tools/cmd/dashboards/export_dashboards.go

@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package main
 
 import (

_beats/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package main
 
 import (

_beats/dev-tools/common.bash

@@ -91,4 +91,8 @@ jenkins_setup() {
   # Workaround for Python virtualenv path being too long.
   export TEMP_PYTHON_ENV=$(mktemp -d)
   export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env"
+
+  # Write cached magefile binaries to workspace to ensure
+  # each run starts from a clean slate.
+  export MAGEFILE_CACHE="${WORKSPACE}/.magefile"
 }

_beats/dev-tools/deploy

@@ -16,9 +16,9 @@ def main():
     check_call("make clean", shell=True)
     print("Done building Docker images.")
     if args.no_snapshot:
-        check_call("make SNAPSHOT=no package-all", shell=True)
+        check_call("make release", shell=True)
     else:
-        check_call("make SNAPSHOT=yes package-all", shell=True)
+        check_call("make snapshot", shell=True)
     print("All done")
 
 if __name__ == "__main__":

_beats/dev-tools/jenkins_ci.ps1

@@ -17,6 +17,10 @@ $env:GOPATH = $env:WORKSPACE
 $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH"
 & gvm --format=powershell $(Get-Content .go-version) | Invoke-Expression
 
+# Write cached magefile binaries to workspace to ensure
+# each run starts from a clean slate.
+$env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
+
 if (Test-Path "$env:beat") {
     cd "$env:beat"
 } else {

_beats/dev-tools/jenkins_release.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euox pipefail
+
+: "${HOME:?Need to set HOME to a non-empty value.}"
+: "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}"
+
+source ./dev-tools/common.bash
+
+jenkins_setup
+
+cleanup() {
+  echo "Running cleanup..."
+  rm -rf $TEMP_PYTHON_ENV
+  echo "Cleanup complete."
+}
+trap cleanup EXIT
+
+make release

_beats/dev-tools/mage/.gitignore

@@ -0,0 +1 @@
+build

_beats/dev-tools/mage/build.go

@@ -0,0 +1,144 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"go/build"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+// BuildArgs are the arguments used for the "build" target and they define how
+// "go build" is invoked.
+type BuildArgs struct {
+	Name       string // Name of binary. (On Windows '.exe' is appended.)
+	OutputDir  string
+	CGO        bool
+	Static     bool
+	Env        map[string]string
+	LDFlags    []string
+	Vars       map[string]string // Vars that are passed as -X key=value with the ldflags.
+	ExtraFlags []string
+}
+
+// DefaultBuildArgs returns the default BuildArgs for use in builds.
+func DefaultBuildArgs() BuildArgs {
+	args := BuildArgs{
+		Name: BeatName,
+		CGO:  build.Default.CgoEnabled,
+		Vars: map[string]string{
+			"github.com/elastic/beats/libbeat/version.buildTime": "{{ date }}",
+			"github.com/elastic/beats/libbeat/version.commit":    "{{ commit }}",
+		},
+	}
+
+	repo, err := GetProjectRepoInfo()
+	if err != nil {
+		panic(errors.Wrap(err, "failed to determine project repo info"))
+	}
+
+	if !repo.IsElasticBeats() {
+		// Assume libbeat is vendored and prefix the variables.
+		prefix := repo.RootImportPath + "/vendor/"
+		prefixedVars := map[string]string{}
+		for k, v := range args.Vars {
+			prefixedVars[prefix+k] = v
+		}
+		args.Vars = prefixedVars
+	}
+
+	return args
+}
+
+// DefaultGolangCrossBuildArgs returns the default BuildArgs for use in
+// cross-builds.
+func DefaultGolangCrossBuildArgs() BuildArgs {
+	args := DefaultBuildArgs()
+	args.Name += "-" + Platform.GOOS + "-" + Platform.Arch
+	args.OutputDir = filepath.Join("build", "golang-crossbuild")
+	if bp, found := BuildPlatforms.Get(Platform.Name); found {
+		args.CGO = bp.Flags.SupportsCGO()
+	}
+	return args
+}
+
+// GolangCrossBuild invokes "go build" inside of the golang-crossbuild Docker
+// environment.
+func GolangCrossBuild(params BuildArgs) error {
+	if os.Getenv("GOLANG_CROSSBUILD") != "1" {
+		return errors.New("Use the crossBuild target. golangCrossBuild can " +
+			"only be executed within the golang-crossbuild docker environment.")
+	}
+
+	defer DockerChown(filepath.Join(params.OutputDir, params.Name+binaryExtension(GOOS)))
+	return Build(params)
+}
+
+// Build invokes "go build" to produce a binary.
+func Build(params BuildArgs) error {
+	fmt.Println(">> build: Building", params.Name)
+
+	binaryName := params.Name + binaryExtension(GOOS)
+
+	if params.OutputDir != "" {
+		if err := os.MkdirAll(params.OutputDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	// Environment
+	env := params.Env
+	if env == nil {
+		env = map[string]string{}
+	}
+	cgoEnabled := "0"
+	if params.CGO {
+		cgoEnabled = "1"
+	}
+	env["CGO_ENABLED"] = cgoEnabled
+
+	// Spec
+	args := []string{
+		"build",
+		"-o",
+		filepath.Join(params.OutputDir, binaryName),
+	}
+	args = append(args, params.ExtraFlags...)
+
+	// ldflags
+	ldflags := params.LDFlags
+	if params.Static {
+		ldflags = append(ldflags, `-extldflags '-static'`)
+	}
+	for k, v := range params.Vars {
+		ldflags = append(ldflags, fmt.Sprintf("-X %v=%v", k, v))
+	}
+	if len(ldflags) > 0 {
+		args = append(args, "-ldflags")
+		args = append(args, MustExpand(strings.Join(ldflags, " ")))
+	}
+
+	log.Println("Adding build environment vars:", env)
+	return sh.RunWith(env, "go", args...)
+}

_beats/dev-tools/mage/clean.go

@@ -0,0 +1,54 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"github.com/magefile/mage/sh"
+)
+
+// DefaultCleanPaths specifies a list of files or paths to recursively delete.
+// The values may contain variables and will be expanded at the time of use.
+var DefaultCleanPaths = []string{
+	"build",
+	"docker-compose.yml.lock",
+	"{{.BeatName}}",
+	"{{.BeatName}}.exe",
+	"{{.BeatName}}.test",
+	"{{.BeatName}}.test.exe",
+	"fields.yml",
+	"_meta/fields.generated.yml",
+	"_meta/kibana.generated",
+	"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
+	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
+}
+
+// Clean clean generated build artifacts.
+func Clean(pathLists ...[]string) error {
+	if len(pathLists) == 0 {
+		pathLists = [][]string{DefaultCleanPaths}
+	}
+	for _, paths := range pathLists {
+		for _, f := range paths {
+			f = MustExpand(f)
+			if err := sh.Rm(f); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}