From 5fc7c229af38e79ee24f2747e8814da0903bc448 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Fri, 14 Dec 2018 10:30:45 -0600 Subject: [PATCH] Add geo fields to `add_host_metadata` processor. (#9392) This lets users add geo data if they have it to the host (cherry picked from commit c6c4a3097753d778ccb4a5fa82d03b8387f4c4bb) --- libbeat/docs/processors-using.asciidoc | 36 +++++- .../add_host_metadata/add_host_metadata.go | 50 ++++++++- .../add_host_metadata_test.go | 103 ++++++++++++++++++ .../processors/add_host_metadata/config.go | 12 ++ 4 files changed, 198 insertions(+), 3 deletions(-) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index c8829ad7f89..0aa3751c36e 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -895,6 +895,14 @@ processors: - add_host_metadata: netinfo.enabled: false cache.ttl: 5m + geo: + name: nyc-dc1-rack1 + location: 40.7128, -74.0060 + continent_name: North America + country_iso_code: US + region_name: New York + region_iso_code: NY + city_name: New York ------------------------------------------------------------------------------- It has the following settings: @@ -903,6 +911,23 @@ It has the following settings: `cache.ttl`:: (Optional) The processor uses an internal cache for the host metadata. This sets the cache expiration time. The default is 5m, negative values disable caching altogether. +`geo.name`:: User definable token to be used for identifying a discrete location. Frequently a datacenter, rack, or similar. + +`geo.location`:: Longitude and latitude in comma separated format. + +`geo.continent_name`:: Name of the continent. + +`geo.country_name`:: Name of the country. + +`geo.region_name`:: Name of the region. + +`geo.city_name`:: Name of the city. + +`geo.country_iso_code`:: ISO country code. + +`geo.region_iso_code`:: ISO region code. + + The `add_host_metadata` processor annotates each event with relevant metadata from the host machine. The fields added to the event are looking as following: @@ -921,7 +946,16 @@ The fields added to the event are looking as following: "name":"Mac OS X" }, "ip": ["192.168.0.1", "10.0.0.1"], - "mac": ["00:25:96:12:34:56", "72:00:06:ff:79:f1"] + "mac": ["00:25:96:12:34:56", "72:00:06:ff:79:f1"], + "geo": { + "continent_name": "North America", + "country_iso_code": "US", + "region_name": "New York", + "region_iso_code": "NY", + "city_name": "New York", + "name": "nyc-dc1-rack1", + "location": "40.7128, -74.0060" + } } } ------------------------------------------------------------------------------- diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 3482a73b300..2741ef35410 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -20,6 +20,7 @@ package add_host_metadata import ( "fmt" "net" + "regexp" "sync" "time" @@ -43,8 +44,9 @@ type addHostMetadata struct { time.Time sync.Mutex } - data common.MapStrPointer - config Config + data common.MapStrPointer + geoData common.MapStr + config Config } const ( @@ -62,6 +64,46 @@ func newHostMetadataProcessor(cfg *common.Config) (processors.Processor, error) data: common.NewMapStrPointer(nil), } p.loadData() + + if config.Geo != nil { + if len(config.Geo.Location) > 0 { + // Regexp matching a number with an optional decimal component + // Valid numbers: '123', '123.23', etc. + latOrLon := `\-?\d+(\.\d+)?` + + // Regexp matching a pair of lat lon coordinates. + // e.g. 40.123, -92.929 + locRegexp := `^\s*` + // anchor to start of string with optional whitespace + latOrLon + // match the latitude + `\s*\,\s*` + // match the separator. optional surrounding whitespace + latOrLon + // match the longitude + `\s*$` //optional whitespace then end anchor + + if m, _ := regexp.MatchString(locRegexp, config.Geo.Location); !m { + return nil, errors.New(fmt.Sprintf("Invalid lat,lon string for add_host_metadata: %s", config.Geo.Location)) + } + } + + geoFields := common.MapStr{ + "name": config.Geo.Name, + "location": config.Geo.Location, + "continent_name": config.Geo.ContinentName, + "country_iso_code": config.Geo.CountryISOCode, + "region_name": config.Geo.RegionName, + "region_iso_code": config.Geo.RegionISOCode, + "city_name": config.Geo.CityName, + } + // Delete any empty values + blankStringMatch := regexp.MustCompile(`^\s*$`) + for k, v := range geoFields { + vStr := v.(string) + if blankStringMatch.MatchString(vStr) { + delete(geoFields, k) + } + } + p.geoData = common.MapStr{"host": common.MapStr{"geo": geoFields}} + } + return p, nil } @@ -73,6 +115,10 @@ func (p *addHostMetadata) Run(event *beat.Event) (*beat.Event, error) { } event.Fields.DeepUpdate(p.data.Get().Clone()) + + if len(p.geoData) > 0 { + event.Fields.DeepUpdate(p.geoData) + } return event, nil } diff --git a/libbeat/processors/add_host_metadata/add_host_metadata_test.go b/libbeat/processors/add_host_metadata/add_host_metadata_test.go index 1c578065c0d..f166a0c9c27 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata_test.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata_test.go @@ -18,10 +18,13 @@ package add_host_metadata import ( + "fmt" "runtime" "testing" "time" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/beat" @@ -100,3 +103,103 @@ func TestConfigNetInfoEnabled(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, v) } + +func TestConfigGeoEnabled(t *testing.T) { + event := &beat.Event{ + Fields: common.MapStr{}, + Timestamp: time.Now(), + } + + config := map[string]interface{}{ + "geo.name": "yerevan-am", + "geo.location": "40.177200, 44.503490", + "geo.continent_name": "Asia", + "geo.country_iso_code": "AM", + "geo.region_name": "Erevan", + "geo.region_iso_code": "AM-ER", + "geo.city_name": "Yerevan", + } + + testConfig, err := common.NewConfigFrom(config) + assert.NoError(t, err) + + p, err := newHostMetadataProcessor(testConfig) + require.NoError(t, err) + + newEvent, err := p.Run(event) + assert.NoError(t, err) + + for configKey, configValue := range config { + t.Run(fmt.Sprintf("Check of %s", configKey), func(t *testing.T) { + v, err := newEvent.GetValue(fmt.Sprintf("host.%s", configKey)) + assert.NoError(t, err) + assert.Equal(t, configValue, v, "Could not find in %s", newEvent) + }) + } +} + +func TestPartialGeo(t *testing.T) { + event := &beat.Event{ + Fields: common.MapStr{}, + Timestamp: time.Now(), + } + + config := map[string]interface{}{ + "geo.name": "yerevan-am", + "geo.city_name": " ", + } + + testConfig, err := common.NewConfigFrom(config) + assert.NoError(t, err) + + p, err := newHostMetadataProcessor(testConfig) + require.NoError(t, err) + + newEvent, err := p.Run(event) + assert.NoError(t, err) + + v, err := newEvent.Fields.GetValue("host.geo.name") + assert.NoError(t, err) + assert.Equal(t, "yerevan-am", v) + + missing := []string{"continent_name", "country_name", "country_iso_code", "region_name", "region_iso_code", "city_name"} + + for _, k := range missing { + path := "host.geo." + k + v, err = newEvent.Fields.GetValue(path) + + assert.Equal(t, common.ErrKeyNotFound, err, "din expect to find %v", path) + } +} + +func TestGeoLocationValidation(t *testing.T) { + locations := []struct { + str string + valid bool + }{ + {"40.177200, 44.503490", true}, + {"-40.177200, -44.503490", true}, + {"garbage", false}, + {"9999999999", false}, + } + + for _, location := range locations { + t.Run(fmt.Sprintf("Location %s validation should be %t", location.str, location.valid), func(t *testing.T) { + + conf, err := common.NewConfigFrom(map[string]interface{}{ + "geo": map[string]interface{}{ + "location": location.str, + }, + }) + require.NoError(t, err) + + _, err = newHostMetadataProcessor(conf) + + if location.valid { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} diff --git a/libbeat/processors/add_host_metadata/config.go b/libbeat/processors/add_host_metadata/config.go index d2454ad3715..339f8b87b2c 100644 --- a/libbeat/processors/add_host_metadata/config.go +++ b/libbeat/processors/add_host_metadata/config.go @@ -25,6 +25,18 @@ import ( type Config struct { NetInfoEnabled bool `config:"netinfo.enabled"` // Add IP and MAC to event CacheTTL time.Duration `config:"cache.ttl"` + Geo *GeoConfig `config:"geo"` +} + +// GeoConfig contains geo configuration data. +type GeoConfig struct { + Name string `config:"name"` + Location string `config:"location"` + ContinentName string `config:"continent_name"` + CountryISOCode string `config:"country_iso_code"` + RegionName string `config:"region_name"` + RegionISOCode string `config:"region_iso_code"` + CityName string `config:"city_name"` } func defaultConfig() Config {