From 4ad2010e9d07048d0b3100cc6f92e9ea5aa2df44 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 12 Mar 2020 14:56:12 +0200 Subject: [PATCH] Cherry-pick #16834 to 7.x: Fix k8s metadata issue (#16976) * Fix k8s metadata issue (#16834) (cherry picked from commit 034ee554ef6849fafea21b72db228f3c7d0fc159) * Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 + .../common/kubernetes/metadata/metadata.go | 10 +++++ .../common/kubernetes/metadata/namespace.go | 26 ++++++++----- .../kubernetes/metadata/namespace_test.go | 23 +++++------ .../common/kubernetes/metadata/node_test.go | 12 +++--- libbeat/common/kubernetes/metadata/pod.go | 22 +---------- .../common/kubernetes/metadata/resource.go | 4 +- .../kubernetes/metadata/resource_test.go | 12 +++--- .../kubernetes/metadata/service_test.go | 39 ++++++++++++------- 9 files changed, 79 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 04f716e89b7..dc74ce08384 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -86,6 +86,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix `NewContainerMetadataEnricher` to use default config for kubernetes module. {pull}16857[16857] +- Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834] *Auditbeat* diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go index d81f79f28d5..a2b474281cf 100644 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ b/libbeat/common/kubernetes/metadata/metadata.go @@ -18,6 +18,8 @@ package metadata import ( + "strings" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/libbeat/common/safemapstr" @@ -40,3 +42,11 @@ func WithFields(key string, value interface{}) FieldOptions { safemapstr.Put(meta, key, value) } } + +// WithLabels FieldOption allows adding labels under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithLabels(kind string) FieldOptions { + return func(meta common.MapStr) { + safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } +} diff --git a/libbeat/common/kubernetes/metadata/namespace.go b/libbeat/common/kubernetes/metadata/namespace.go index 92dfbc55b2a..dd3011a6c47 100644 --- a/libbeat/common/kubernetes/metadata/namespace.go +++ b/libbeat/common/kubernetes/metadata/namespace.go @@ -60,17 +60,16 @@ func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) common.M return nil } - obj, ok, _ := n.store.GetByKey(name) - if !ok { - return nil - } + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } - no, ok := obj.(*kubernetes.Namespace) - if !ok { - return nil + return n.Generate(no, opts...) } - return n.Generate(no, opts...) + return nil } func flattenMetadata(in common.MapStr) common.MapStr { @@ -84,7 +83,6 @@ func flattenMetadata(in common.MapStr) common.MapStr { if !ok { return nil } - for k, v := range fields { if k == "name" { out[resource] = v @@ -93,5 +91,15 @@ func flattenMetadata(in common.MapStr) common.MapStr { } } + rawLabels, err := in.GetValue("labels") + if err != nil { + return out + } + labels, ok := rawLabels.(common.MapStr) + if !ok { + return out + } + out[resource+"_labels"] = labels + return out } diff --git a/libbeat/common/kubernetes/metadata/namespace_test.go b/libbeat/common/kubernetes/metadata/namespace_test.go index 5ac460d7ca7..40763887197 100644 --- a/libbeat/common/kubernetes/metadata/namespace_test.go +++ b/libbeat/common/kubernetes/metadata/namespace_test.go @@ -58,17 +58,18 @@ func TestNamespace_Generate(t *testing.T) { }, }, // Use this for 8.0 - /*output: common.MapStr{ - "namespace": common.MapStr{ - "name": "obj", - "uid": uid, - "labels": common.MapStr{ - "foo": "bar", + /* + output: common.MapStr{ + "namespace": common.MapStr{ + "name": name, + "uid": uid, + "labels": common.MapStr{ + "foo": "bar", + }, }, - }, - },*/ + },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", @@ -114,7 +115,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { /* output: common.MapStr{ "namespace": common.MapStr{ - "name": "obj", + "name": name, "uid": uid, "labels": common.MapStr{ "foo": "bar", @@ -122,7 +123,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { }, },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go index 226e84710c2..100db49c6d8 100644 --- a/libbeat/common/kubernetes/metadata/node_test.go +++ b/libbeat/common/kubernetes/metadata/node_test.go @@ -61,9 +61,9 @@ func TestNode_Generate(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, @@ -106,9 +106,9 @@ func TestNode_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index c25db1f7a76..9234f54533b 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,11 +49,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) - // TODO: remove this call when moving to 8.0 - out = p.exportPodLabelsAndAnnotations(out) if p.node != nil { - meta := p.node.GenerateFromName(po.Spec.NodeName) + meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node")) if meta != nil { out.Put("node", meta["node"]) } else { @@ -91,21 +89,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } - -func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr { - labels, err := in.GetValue("pod.labels") - if err != nil { - return in - } - in.Put("labels", labels) - in.Delete("pod.labels") - - annotations, err := in.GetValue("pod.annotations") - if err != nil { - return in - } - in.Put("annotations", annotations) - in.Delete("pod.annotations") - - return in -} diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go index e4f4b8ca028..f8f986fe42c 100644 --- a/libbeat/common/kubernetes/metadata/resource.go +++ b/libbeat/common/kubernetes/metadata/resource.go @@ -98,11 +98,11 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie } if len(labelMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".labels", labelMap) + safemapstr.Put(meta, "labels", labelMap) } if len(annotationsMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".annotations", annotationsMap) + safemapstr.Put(meta, "annotations", annotationsMap) } for _, option := range options { diff --git a/libbeat/common/kubernetes/metadata/resource_test.go b/libbeat/common/kubernetes/metadata/resource_test.go index 6df542742f0..f4c25ecbad5 100644 --- a/libbeat/common/kubernetes/metadata/resource_test.go +++ b/libbeat/common/kubernetes/metadata/resource_test.go @@ -62,9 +62,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -99,9 +99,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go index a574f1a7b7a..0fc06af3a0b 100644 --- a/libbeat/common/kubernetes/metadata/service_test.go +++ b/libbeat/common/kubernetes/metadata/service_test.go @@ -64,9 +64,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -101,9 +101,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -153,9 +153,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -190,9 +190,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -262,10 +262,19 @@ func TestService_GenerateWithNamespace(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, + "labels": common.MapStr{ + "foo": "bar", + }, + // Use this for 8.0 + /* + "namespace": common.MapStr{ + "name": "default", + "uid": uid, + "labels": common.MapStr{ + "nskey": "nsvalue", + }, + },*/ "namespace": "default", "namespace_uid": uid, "namespace_labels": common.MapStr{