Merge branch 'main' into draft/sql/fetch_from_all_dbs

CHANGELOG.next.asciidoc

@@ -71,9 +71,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640]
 - Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820]
 - Do not print context cancelled error message when running under agent {pull}36006[36006]
-
-
 - Fix recovering from invalid output configuration when running under Elastic-Agent {pull}36016[36016]
+- Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928]
 
 *Auditbeat*
 

libbeat/common/streambuf/streambuf.go

@@ -155,7 +155,9 @@ func (b *Buffer) doAppend(data []byte, retainable bool, newCap int) error {
 				b.data = tmp
 			}
 		}
-		b.data = append(b.data, data...)
+		tBuf := bytes.NewBuffer(b.data)
+		tBuf.Write(data)
+		b.data = tBuf.Bytes()
 	}
 	b.available += len(data)
 

metricbeat/internal/sysinit/init.go

@@ -19,6 +19,7 @@ package sysinit
 
 import (
 	"flag"
+	"fmt"
 	"sync"
 
 	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
@@ -39,21 +40,24 @@ type HostFSConfig struct {
 	HostFS string `config:"hostfs"`
 }
 
-// MetricbeatHostFSConfig
+// MetricbeatHostFSConfig carries config information for the hostfs setting
 type MetricbeatHostFSConfig struct {
 	HostFS string `config:"system.hostfs"`
 }
 
-// Init either the system or linux module. This will produce different modules depending on if we're running under agent or not.
+// InitSystemModule initializes either either the system or linux module. This will produce different modules depending on if we're running under agent or not.
 func InitSystemModule(base mb.BaseModule) (mb.Module, error) {
 	// common code for the base use case of `hostfs` being set at the module-level
 	logger := logp.L()
-	hostfs, userSet := findConfigValue(base)
+	hostfs, userSet, err := findConfigValue(base)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching config value: %w", err)
+	}
 	if fleetmode.Enabled() {
 		logger.Infof("initializing HostFS values under agent: %s", hostfs)
 		return fleetInit(base, hostfs, userSet)
 	}
-	return metricbeatInit(base, hostfs, userSet)
+	return metricbeatInit(base, hostfs)
 }
 
 func fleetInit(base mb.BaseModule, modulepath string, moduleSet bool) (mb.Module, error) {
@@ -71,12 +75,11 @@ func fleetInit(base mb.BaseModule, modulepath string, moduleSet bool) (mb.Module
 }
 
 // Deal with the legacy configs available to metricbeat
-func metricbeatInit(base mb.BaseModule, modulePath string, moduleSet bool) (mb.Module, error) {
+func metricbeatInit(base mb.BaseModule, modulePath string) (mb.Module, error) {
 	var hostfs = modulePath
 	var userSet bool
 	// allow the CLI to override other settings
 	if hostfsCLI != nil && *hostfsCLI != "" {
-		cfgwarn.Deprecate("8.0.0", "The --system.hostfs flag will be removed in the future and replaced by a config value.")
 		hostfs = *hostfsCLI
 		userSet = true
 	}
@@ -91,22 +94,29 @@ func metricbeatInit(base mb.BaseModule, modulePath string, moduleSet bool) (mb.M
 // A user can supply either `system.hostfs` or `hostfs`.
 // In additon, we will probably want to change Integration Config values to `hostfs` as well.
 // We need to figure out which one we got, if any.
-func findConfigValue(base mb.BaseModule) (string, bool) {
+// Returns false if no config value was set
+func findConfigValue(base mb.BaseModule) (string, bool, error) {
 	partialConfig := HostFSConfig{}
-	base.UnpackConfig(&partialConfig)
+	err := base.UnpackConfig(&partialConfig)
+	if err != nil {
+		return "", false, fmt.Errorf("error unpacking hostfs config: %w", err)
+	}
 	// if the newer value is set, just use that.
 	if partialConfig.HostFS != "" {
-		return partialConfig.HostFS, true
+		return partialConfig.HostFS, true, nil
 	}
 
 	legacyConfig := MetricbeatHostFSConfig{}
-	base.UnpackConfig(&legacyConfig)
+	err = base.UnpackConfig(&legacyConfig)
+	if err != nil {
+		return "", false, fmt.Errorf("error unpacking legacy config: %w", err)
+	}
 	if legacyConfig.HostFS != "" {
 		cfgwarn.Deprecate("8.0.0", "The system.hostfs config value will be removed, use `hostfs` from within the module config.")
 		// Only fallback to this if the user didn't set anything else
-		return legacyConfig.HostFS, true
+		return legacyConfig.HostFS, true, nil
 	}
 
-	return "/", false
+	return "/", false, nil
 
 }