From 30dfa052aa1b0ba6fc63271869b7567f6e3370d0 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 4 Dec 2018 16:17:24 +0100 Subject: [PATCH] Add cache.ttl to add_host_metadata (#9359) Fix: Refresh host info when cache expires. Add config parameter `cache.ttl` to control cache expiration (default 5m). --- CHANGELOG.asciidoc | 2 + libbeat/docs/processors-using.asciidoc | 5 ++- .../add_host_metadata/add_host_metadata.go | 38 +++++++++++-------- .../processors/add_host_metadata/config.go | 8 +++- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 14b1c19cbdd..c1b5f38c045 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d - Fix autodiscover configurations stopping when metadata is missing. {pull}8851[8851] - Log events at the debug level when dropped by encoding problems. {pull}9251[9251] +- Refresh host metadata in add_host_metadata. {pull}9359[9359] *Auditbeat* @@ -70,6 +71,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d - Unify dashboard exporter tools. {pull}9097[9097] - Use _doc as document type of the Elasticsearch major version is 7. {pull}9056[9056] +- Add cache.ttl to add_host_metadata. {pull}9359[9359] *Auditbeat* diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 8b92f23d833..449b9d3b22a 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -894,12 +894,15 @@ beta[] processors: - add_host_metadata: netinfo.enabled: false + cache.ttl: 5m ------------------------------------------------------------------------------- It has the following settings: `netinfo.enabled`:: (Optional) Default false. Include IP addresses and MAC addresses as fields host.ip and host.mac +`cache.ttl`:: (Optional) The processor uses an internal cache for the host metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether. + The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event are looking as following: @@ -923,8 +926,6 @@ The fields added to the event are looking as following: } ------------------------------------------------------------------------------- -NOTE: The host information is refreshed every 5 minutes. - [[dissect]] === Dissect strings diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 5127d8b7e53..3482a73b300 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -32,7 +32,6 @@ import ( "github.com/elastic/beats/libbeat/metric/system/host" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/go-sysinfo" - "github.com/elastic/go-sysinfo/types" ) func init() { @@ -40,7 +39,6 @@ func init() { } type addHostMetadata struct { - info types.HostInfo lastUpdate struct { time.Time sync.Mutex @@ -50,8 +48,7 @@ type addHostMetadata struct { } const ( - processorName = "add_host_metadata" - cacheExpiration = time.Minute * 5 + processorName = "add_host_metadata" ) func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) { @@ -60,12 +57,7 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) } - h, err := sysinfo.Host() - if err != nil { - return nil, err - } p := &addHostMetadata{ - info: h.Info(), config: config, data: common.NewMapStrPointer(nil), } @@ -75,28 +67,41 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) // Run enriches the given event with the host meta data func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { - p.loadData() + err := p.loadData() + if err != nil { + return nil, err + } + event.Fields.DeepUpdate(p.data.Get().Clone()) return event, nil } func (p *addHostMetadata) expired() bool { + if p.config.CacheTTL <= 0 { + return true + } + p.lastUpdate.Lock() defer p.lastUpdate.Unlock() - if p.lastUpdate.Add(cacheExpiration).After(time.Now()) { + if p.lastUpdate.Add(p.config.CacheTTL).After(time.Now()) { return false } p.lastUpdate.Time = time.Now() return true } -func (p *addHostMetadata) loadData() { +func (p *addHostMetadata) loadData() error { if !p.expired() { - return + return nil + } + + h, err := sysinfo.Host() + if err != nil { + return err } - data := host.MapHostInfo(p.info) + data := host.MapHostInfo(h.Info()) if p.config.NetInfoEnabled { // IP-address and MAC-address var ipList, hwList, err = p.getNetInfo() @@ -113,6 +118,7 @@ func (p *addHostMetadata) loadData() { } p.data.Set(data) + return nil } func (p *addHostMetadata) getNetInfo() ([]string, []string, error) { @@ -161,6 +167,6 @@ func (p *addHostMetadata) getNetInfo() ([]string, []string, error) { } func (p *addHostMetadata) String() string { - return fmt.Sprintf("%v=[netinfo.enabled=[%v]]", - processorName, p.config.NetInfoEnabled) + return fmt.Sprintf("%v=[netinfo.enabled=[%v], cache.ttl=[%v]]", + processorName, p.config.NetInfoEnabled, p.config.CacheTTL) } diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index 30f2e293b69..d2454ad3715 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -17,13 +17,19 @@ package add_host_metadata +import ( + "time" +) + // Config for add_host_metadata processor. type Config struct { - NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event + NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event + CacheTTL time.Duration `config:"cache.ttl"` } func defaultConfig() Config { return Config{ NetInfoEnabled: false, + CacheTTL: 5 * time.Minute, } }