[Auditbeat] Package: Auto-detect package directories (elastic#12289)

Changes the `system/package` dataset to select the package manager based on which package directories exist: `/var/lib/dpkg`, `/var/lib/rpm`, or `/usr/local/Cellar`.

If we find no directories, we log a warning once and continue checking. We do not abort Auditbeat's launch.

(cherry picked from commit afbe070)

CHANGELOG.next.asciidoc

@@ -88,6 +88,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Package dataset: Fixed a crash inside librpm after Auditbeat has been running for a while. {issue}12147[12147] {pull}12168[12168]
 - Fix formatting of config files on macOS and Windows. {pull}12148[12148]
 - Fix direction of incoming IPv6 sockets. {pull}12248[12248]
+- Package dataset: Auto-detect package directories. {pull}12289[12289]
 
 *Filebeat*
 

x-pack/auditbeat/module/system/package/package.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -29,22 +30,13 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/x-pack/auditbeat/cache"
 	"github.com/elastic/beats/x-pack/auditbeat/module/system"
-	"github.com/elastic/go-sysinfo"
-	"github.com/elastic/go-sysinfo/types"
 )
 
 const (
 	moduleName    = "system"
 	metricsetName = "package"
 	namespace     = "system.audit.package"
 
-	redhat = "redhat"
-	suse   = "suse"
-	debian = "debian"
-	darwin = "darwin"
-
-	dpkgStatusFile = "/var/lib/dpkg/status"
-
 	bucketName              = "package.v1"
 	bucketKeyPackages       = "packages"
 	bucketKeyStateTimestamp = "state_timestamp"
@@ -54,6 +46,8 @@ const (
 )
 
 var (
+	rpmPath            = "/var/lib/rpm"
+	dpkgPath           = "/var/lib/dpkg"
 	homebrewCellarPath = "/usr/local/Cellar"
 )
 
@@ -96,7 +90,8 @@ type MetricSet struct {
 	cache     *cache.Cache
 	bucket    datastore.Bucket
 	lastState time.Time
-	osFamily  string
+
+	suppressNoPackageWarnings bool
 }
 
 // Package represents information for a package.
@@ -169,20 +164,6 @@ func (pkg Package) entityID(hostID string) string {
 	return h.Sum()
 }
 
-func getOS() (*types.OSInfo, error) {
-	host, err := sysinfo.Host()
-	if err != nil {
-		return nil, errors.Wrap(err, "error getting the OS")
-	}
-
-	hostInfo := host.Info()
-	if hostInfo.OS == nil {
-		return nil, errors.New("no host info")
-	}
-
-	return hostInfo.OS, nil
-}
-
 // New constructs a new MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The %v/%v dataset is beta", moduleName, metricsetName)
@@ -205,26 +186,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		bucket:          bucket,
 	}
 
-	osInfo, err := getOS()
-	if err != nil {
-		return nil, errors.Wrap(err, "error determining operating system")
-	}
-	ms.osFamily = osInfo.Family
-	switch osInfo.Family {
-	case redhat, suse:
-		// ok
-	case debian:
-		if _, err := os.Stat(dpkgStatusFile); err != nil {
-			return nil, errors.Wrapf(err, "error looking up %s", dpkgStatusFile)
-		}
-	case darwin:
-		if _, err := os.Stat(homebrewCellarPath); err != nil {
-			ms.log.Errorf("Homebrew does not seem to be installed. Will keep trying. Error: %v", err)
-		}
-	default:
-		return nil, fmt.Errorf("this metricset does not support OS family %v", osInfo.Family)
-	}
-
 	// Load from disk: Time when state was last sent
 	err = bucket.Load(bucketKeyStateTimestamp, func(blob []byte) error {
 		if len(blob) > 0 {
@@ -285,11 +246,10 @@ func (ms *MetricSet) Fetch(report mb.ReporterV2) {
 func (ms *MetricSet) reportState(report mb.ReporterV2) error {
 	ms.lastState = time.Now()
 
-	packages, err := ms.getPackages(ms.osFamily)
+	packages, err := ms.getPackages()
 	if err != nil {
 		return errors.Wrap(err, "failed to get packages")
 	}
-	ms.log.Debugf("Found %v packages", len(packages))
 
 	stateID, err := uuid.NewV4()
 	if err != nil {
@@ -319,11 +279,10 @@ func (ms *MetricSet) reportState(report mb.ReporterV2) error {
 
 // reportChanges detects and reports any changes to installed packages on this system since the last call.
 func (ms *MetricSet) reportChanges(report mb.ReporterV2) error {
-	packages, err := ms.getPackages(ms.osFamily)
+	packages, err := ms.getPackages()
 	if err != nil {
 		return errors.Wrap(err, "failed to get packages")
 	}
-	ms.log.Debugf("Found %v packages", len(packages))
 
 	newInCache, missingFromCache := ms.cache.DiffAndUpdateCache(convertToCacheable(packages))
 	newPackages := convertToPackage(newInCache)
@@ -473,35 +432,68 @@ func (ms *MetricSet) savePackagesToDisk(packages []*Package) error {
 	return nil
 }
 
-func (ms *MetricSet) getPackages(osFamily string) (packages []*Package, err error) {
-	switch osFamily {
-	case redhat, suse:
-		packages, err = listRPMPackages()
+func (ms *MetricSet) getPackages() (packages []*Package, err error) {
+	var foundPackageManager bool
+
+	_, err = os.Stat(rpmPath)
+	if err == nil {
+		foundPackageManager = true
+
+		rpmPackages, err := listRPMPackages()
 		if err != nil {
 			return nil, errors.Wrap(err, "error getting RPM packages")
 		}
-	case debian:
-		packages, err = listDebPackages()
+		ms.log.Debugf("RPM packages: %v", len(rpmPackages))
+
+		packages = append(packages, rpmPackages...)
+	} else if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "error opening %v", rpmPath)
+	}
+
+	_, err = os.Stat(dpkgPath)
+	if err == nil {
+		foundPackageManager = true
+
+		dpkgPackages, err := listDebPackages()
 		if err != nil {
 			return nil, errors.Wrap(err, "error getting DEB packages")
 		}
-	case darwin:
-		packages, err = listBrewPackages()
+		ms.log.Debugf("DEB packages: %v", len(dpkgPackages))
+
+		packages = append(packages, dpkgPackages...)
+	} else if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "error opening %v", dpkgPath)
+	}
+
+	_, err = os.Stat(homebrewCellarPath)
+	if err == nil {
+		foundPackageManager = true
+
+		homebrewPackages, err := listBrewPackages()
 		if err != nil {
-			if os.IsNotExist(err) {
-				ms.log.Debugf("Homebrew not installed: %v", err)
-			} else {
-				return nil, errors.Wrap(err, "error getting Homebrew packages")
-			}
+			return nil, errors.Wrap(err, "error getting Homebrew packages")
 		}
-	default:
-		return nil, errors.Errorf("unknown OS %v - this should not have happened", osFamily)
+		ms.log.Debugf("Homebrew packages: %v", len(homebrewPackages))
+
+		packages = append(packages, homebrewPackages...)
+	} else if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "error opening %v", homebrewCellarPath)
+	}
+
+	if !foundPackageManager && !ms.suppressNoPackageWarnings {
+		ms.log.Warnf("No supported package managers found. None of %v, %v, %v exist.",
+			rpmPath, dpkgPath, homebrewCellarPath)
+
+		// Only warn once at the start of Auditbeat.
+		ms.suppressNoPackageWarnings = true
 	}
 
 	return packages, nil
 }
 
 func listDebPackages() ([]*Package, error) {
+	dpkgStatusFile := filepath.Join(dpkgPath, "status")
+
 	file, err := os.Open(dpkgStatusFile)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error opening %s", dpkgStatusFile)

x-pack/auditbeat/module/system/package/rpm_linux_test.go

@@ -7,21 +7,20 @@
 package pkg
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRPMPackages(t *testing.T) {
-	os, err := getOS()
-	if err != nil {
+	_, err := os.Stat(rpmPath)
+	if os.IsNotExist(err) {
+		t.Skipf("RPM test only on systems where %v exists", rpmPath)
+	} else if err != nil {
 		t.Fatal(err)
 	}
 
-	if os.Family != "redhat" {
-		t.Skip("RPM test only on Redhat systems")
-	}
-
 	// Control using the exec command
 	packagesExpected, err := rpmPackagesByExec()
 	if err != nil {
@@ -34,5 +33,4 @@ func TestRPMPackages(t *testing.T) {
 	}
 
 	assert.EqualValues(t, packagesExpected, packages)
-
 }