From 8fd725dc2ed295c8b52eb290b4650c101f0aba19 Mon Sep 17 00:00:00 2001 From: Rob E Date: Thu, 22 Nov 2018 22:45:52 +1000 Subject: [PATCH 1/3] kubernetes processor use correct os path separator When appending a trailing slash to the log path directory, it needs to use the correct PathSeperator for the given OS. https://github.com/elastic/beats/issues/9196 --- filebeat/processor/add_kubernetes_metadata/matchers.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index bac49354354..6489d46c8fe 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -19,6 +19,7 @@ package add_kubernetes_metadata import ( "fmt" + "os" "strings" "github.com/elastic/beats/libbeat/common" @@ -59,8 +60,8 @@ func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, err } logPath := config.LogsPath - if logPath[len(logPath)-1:] != "/" { - logPath = logPath + "/" + if logPath[len(logPath)-1:] != string(os.PathSeparator) { + logPath = logPath + string(os.PathSeparator) } resourceType := config.ResourceType From 3a5b30d6dc923fe235f34e916ee8d804121198f8 Mon Sep 17 00:00:00 2001 From: Robert Erez Date: Tue, 27 Nov 2018 20:33:32 +1000 Subject: [PATCH 2/3] Added windows tests for kubernetes processor matches --- .../add_kubernetes_metadata/matchers.go | 37 +++++++++++++++---- .../add_kubernetes_metadata/matchers_test.go | 35 ++++++++++++++++-- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index 6489d46c8fe..2688850851b 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -20,6 +20,7 @@ package add_kubernetes_metadata import ( "fmt" "os" + "runtime" "strings" "github.com/elastic/beats/libbeat/common" @@ -39,6 +40,7 @@ func init() { } const LogPathMatcherName = "logs_path" +const PathSeparator = string(os.PathSeparator) type LogPathMatcher struct { LogsPath string @@ -50,7 +52,7 @@ func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, err LogsPath string `config:"logs_path"` ResourceType string `config:"resource_type"` }{ - LogsPath: "/var/lib/docker/containers/", + LogsPath: defaultLogPath(), ResourceType: "container", } @@ -60,8 +62,8 @@ func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, err } logPath := config.LogsPath - if logPath[len(logPath)-1:] != string(os.PathSeparator) { - logPath = logPath + string(os.PathSeparator) + if logPath[len(logPath)-1:] != PathSeparator { + logPath = logPath + PathSeparator } resourceType := config.ResourceType @@ -93,10 +95,10 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { if f.ResourceType == "pod" { // Specify a pod resource type when manually mounting log volumes and they end up under "/var/lib/kubelet/pods/" // This will extract only the pod UID, which offers less granularity of metadata when compared to the container ID - if strings.HasPrefix(f.LogsPath, "/var/lib/kubelet/pods/") && strings.HasSuffix(source, ".log") { - pathDirs := strings.Split(source, "/") + if strings.HasPrefix(f.LogsPath, PodLogsPath()) && strings.HasSuffix(source, ".log") { + pathDirs := strings.Split(source, PathSeparator) if len(pathDirs) > podUIDPos { - podUID := strings.Split(source, "/")[podUIDPos] + podUID := strings.Split(source, PathSeparator)[podUIDPos] logp.Debug("kubernetes", "Using pod uid: %s", podUID) return podUID @@ -107,7 +109,7 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { } else { // In case of the Kubernetes log path "/var/log/containers/", // the container ID will be located right before the ".log" extension. - if strings.HasPrefix(f.LogsPath, "/var/log/containers/") && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 { + if strings.HasPrefix(f.LogsPath, ContainerLogsPath()) && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 { containerIDEnd := sourceLen - 4 cid := source[containerIDEnd-containerIdLen : containerIDEnd] logp.Debug("kubernetes", "Using container id: %s", cid) @@ -128,3 +130,24 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { return "" } + +func defaultLogPath() string { + if runtime.GOOS == "windows" { + return "C:\\ProgramData\\Docker\\containers" + } + return "/var/lib/docker/containers/" +} + +func PodLogsPath() string { + if runtime.GOOS == "windows" { + return "C:\\var\\lib\\kubelet\\pods\\" + } + return "/var/lib/kubelet/pods/" +} + +func ContainerLogsPath() string { + if runtime.GOOS == "windows" { + return "C:\\var\\log\\containers\\" + } + return "/var/log/containers/" +} diff --git a/filebeat/processor/add_kubernetes_metadata/matchers_test.go b/filebeat/processor/add_kubernetes_metadata/matchers_test.go index 2a8b0bf89d6..4cf45e72e31 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers_test.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers_test.go @@ -19,6 +19,7 @@ package add_kubernetes_metadata import ( "fmt" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -55,29 +56,55 @@ func TestLogsPathMatcher_InvalidSource3(t *testing.T) { func TestLogsPathMatcher_VarLibDockerContainers(t *testing.T) { cfgLogsPath := "" // use the default matcher configuration - source := fmt.Sprintf("/var/lib/docker/containers/%s/%s-json.log", cid, cid) + + path := "/var/lib/docker/containers/%s/%s-json.log" + if runtime.GOOS == "windows" { + path = "C:\\ProgramData\\Docker\\containers\\%s\\%s-json.log" + } + + source := fmt.Sprintf(path, cid, cid) + expectedResult := cid executeTest(t, cfgLogsPath, source, expectedResult) } func TestLogsPathMatcher_VarLogContainers(t *testing.T) { cfgLogsPath := "/var/log/containers/" - source := fmt.Sprintf("/var/log/containers/kube-proxy-4d7nt_kube-system_kube-proxy-%s.log", cid) + sourcePath := "/var/log/containers/kube-proxy-4d7nt_kube-system_kube-proxy-%s.log" + if runtime.GOOS == "windows" { + cfgLogsPath = "C:\\var\\log\\containers\\" + sourcePath = "C:\\var\\log\\containers\\kube-proxy-4d7nt_kube-system_kube-proxy-%s.log" + } + + source := fmt.Sprintf(sourcePath, cid) expectedResult := cid executeTest(t, cfgLogsPath, source, expectedResult) } func TestLogsPathMatcher_AnotherLogDir(t *testing.T) { cfgLogsPath := "/var/log/other/" - source := fmt.Sprintf("/var/log/other/%s.log", cid) + sourcePath := "/var/log/other/%s.log" + if runtime.GOOS == "windows" { + cfgLogsPath = "C:\\var\\log\\other\\" + sourcePath = "C:\\var\\log\\other\\%s.log" + } + + source := fmt.Sprintf(sourcePath, cid) expectedResult := cid executeTest(t, cfgLogsPath, source, expectedResult) } func TestLogsPathMatcher_VarLibKubeletPods(t *testing.T) { cfgLogsPath := "/var/lib/kubelet/pods/" + sourcePath := "/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/applogs/server.log" cfgResourceType := "pod" - source := fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/applogs/server.log", puid) + + if runtime.GOOS == "windows" { + cfgLogsPath = "C:\\var\\lib\\kubelet\\pods\\" + sourcePath = "C:\\var\\lib\\kubelet\\pods\\%s\\volumes\\kubernetes.io~empty-dir\\applogs\\server.log" + } + + source := fmt.Sprintf(sourcePath, puid) expectedResult := puid executeTestWithResourceType(t, cfgLogsPath, cfgResourceType, source, expectedResult) } From b176ab55b78bf004e4b142c443a5134d5b2d15ea Mon Sep 17 00:00:00 2001 From: Robert Erez Date: Tue, 27 Nov 2018 20:49:45 +1000 Subject: [PATCH 3/3] Unexport local functions in k8s matcher --- .../add_kubernetes_metadata/matchers.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index 2688850851b..125425971ec 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -40,7 +40,7 @@ func init() { } const LogPathMatcherName = "logs_path" -const PathSeparator = string(os.PathSeparator) +const pathSeparator = string(os.PathSeparator) type LogPathMatcher struct { LogsPath string @@ -62,8 +62,8 @@ func newLogsPathMatcher(cfg common.Config) (add_kubernetes_metadata.Matcher, err } logPath := config.LogsPath - if logPath[len(logPath)-1:] != PathSeparator { - logPath = logPath + PathSeparator + if logPath[len(logPath)-1:] != pathSeparator { + logPath = logPath + pathSeparator } resourceType := config.ResourceType @@ -95,10 +95,10 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { if f.ResourceType == "pod" { // Specify a pod resource type when manually mounting log volumes and they end up under "/var/lib/kubelet/pods/" // This will extract only the pod UID, which offers less granularity of metadata when compared to the container ID - if strings.HasPrefix(f.LogsPath, PodLogsPath()) && strings.HasSuffix(source, ".log") { - pathDirs := strings.Split(source, PathSeparator) + if strings.HasPrefix(f.LogsPath, podLogsPath()) && strings.HasSuffix(source, ".log") { + pathDirs := strings.Split(source, pathSeparator) if len(pathDirs) > podUIDPos { - podUID := strings.Split(source, PathSeparator)[podUIDPos] + podUID := strings.Split(source, pathSeparator)[podUIDPos] logp.Debug("kubernetes", "Using pod uid: %s", podUID) return podUID @@ -109,7 +109,7 @@ func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { } else { // In case of the Kubernetes log path "/var/log/containers/", // the container ID will be located right before the ".log" extension. - if strings.HasPrefix(f.LogsPath, ContainerLogsPath()) && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 { + if strings.HasPrefix(f.LogsPath, containerLogsPath()) && strings.HasSuffix(source, ".log") && sourceLen >= containerIdLen+4 { containerIDEnd := sourceLen - 4 cid := source[containerIDEnd-containerIdLen : containerIDEnd] logp.Debug("kubernetes", "Using container id: %s", cid) @@ -138,14 +138,14 @@ func defaultLogPath() string { return "/var/lib/docker/containers/" } -func PodLogsPath() string { +func podLogsPath() string { if runtime.GOOS == "windows" { return "C:\\var\\lib\\kubelet\\pods\\" } return "/var/lib/kubelet/pods/" } -func ContainerLogsPath() string { +func containerLogsPath() string { if runtime.GOOS == "windows" { return "C:\\var\\log\\containers\\" }