Stop using mage:import in custom beats 7.4 (#14187)

`mage:import` doesn't take into account vendorized dependencies, this is a
problen for custom beats, because by default they have libbeat as a
vendorized dependency. The solution would be to use go modules, but in
the meantime lets remove the use of `mage:import` from the magefiles
intended to be used by custom beats.

Revert part of #13239 related to custom beats.

CHANGELOG-developer.next.asciidoc

@@ -24,6 +24,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 
 ==== Bugfixes
 
+- Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}[]
+
 ==== Added
 
 - Metricset generator generates beta modules by default now. {pull}10657[10657]

generator/beat/{beat}/magefile.go.tmpl → generator/beat/{beat}/magefile.go

@@ -28,9 +28,7 @@ import (
 	"github.com/magefile/mage/sh"
 
 	devtools "github.com/elastic/beats/dev-tools/mage"
-
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/common"
+	"github.com/elastic/beats/dev-tools/mage/target/common"
 )
 
 func init() {
@@ -66,6 +64,11 @@ func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
 
+// Clean cleans all generated files and build artifacts.
+func Clean() error {
+	return devtools.Clean()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.
@@ -113,3 +116,14 @@ func GoTestIntegration(ctx context.Context) error {
 func Config() error {
 	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
 }
+
+// Check formats code, updates generated content, check for common errors, and
+// checks for any modified files.
+func Check() {
+	common.Check()
+}
+
+// Fmt formats source code (.go and .py) and adds license headers.
+func Fmt() {
+	common.Fmt()
+}

generator/metricbeat/{beat}/magefile.go

@@ -20,27 +20,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 
 	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/dev-tools/mage/target/common"
 	metricbeat "github.com/elastic/beats/metricbeat/scripts/mage"
-
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/common"
-	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/build"
-	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/pkg"
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/test"
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/unittest"
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/integtest"
 )
 
 func init() {
@@ -54,6 +43,37 @@ func CollectAll() {
 	mg.Deps(CollectDocs, FieldsDocs)
 }
 
+// Build builds the Beat binary.
+func Build() error {
+	return devtools.Build(devtools.DefaultBuildArgs())
+}
+
+// GolangCrossBuild build the Beat binary inside of the golang-builder.
+// Do not use directly, use crossBuild instead.
+func GolangCrossBuild() error {
+	return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs())
+}
+
+// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).
+func BuildGoDaemon() error {
+	return devtools.BuildGoDaemon()
+}
+
+// CrossBuild cross-builds the beat for all target platforms.
+func CrossBuild() error {
+	return devtools.CrossBuild()
+}
+
+// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker.
+func CrossBuildGoDaemon() error {
+	return devtools.CrossBuildGoDaemon()
+}
+
+// Clean cleans all generated files and build artifacts.
+func Clean() error {
+	return devtools.Clean()
+}
+
 // Package packages the Beat for distribution.
 // Use SNAPSHOT=true to build snapshots.
 // Use PLATFORMS to control the target platforms.
@@ -64,8 +84,13 @@ func Package() {
 	devtools.UseCommunityBeatPackaging()
 
 	mg.Deps(Update)
-	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
-	mg.SerialDeps(devtools.Package, pkg.PackageTest)
+	mg.Deps(CrossBuild, CrossBuildGoDaemon)
+	mg.SerialDeps(devtools.Package, TestPackages)
+}
+
+// TestPackages tests the generated packages (i.e. file modes, owners, groups).
+func TestPackages() error {
+	return devtools.TestPackages()
 }
 
 // Update updates the generated files (aka make update).
@@ -96,6 +121,20 @@ func CollectDocs() error {
 	return metricbeat.CollectDocs()
 }
 
+// GoTestUnit executes the Go unit tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoTestUnit(ctx context.Context) error {
+	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
+}
+
+// GoTestIntegration executes the Go integration tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoTestIntegration(ctx context.Context) error {
+	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+}
+
 // Config generates both the short/reference/docker configs.
 func Config() error {
 	customDeps := devtools.ConfigFileParams{
@@ -106,3 +145,14 @@ func Config() error {
 	}
 	return devtools.Config(devtools.AllConfigTypes, customDeps, ".")
 }
+
+// Check formats code, updates generated content, check for common errors, and
+// checks for any modified files.
+func Check() {
+	common.Check()
+}
+
+// Fmt formats source code (.go and .py) and adds license headers.
+func Fmt() {
+	common.Fmt()
+}