diff --git a/pkg/integrations/v4/logs/cfg.go b/pkg/integrations/v4/logs/cfg.go index 57fe42acf..3deb12536 100644 --- a/pkg/integrations/v4/logs/cfg.go +++ b/pkg/integrations/v4/logs/cfg.go @@ -89,18 +89,19 @@ type YAML struct { // LogCfg logging integration config from customer defined YAML. type LogCfg struct { - Name string `yaml:"name"` - File string `yaml:"file"` // ... - MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines. - Systemd string `yaml:"systemd"` // ... - Pattern string `yaml:"pattern"` - Attributes map[string]string `yaml:"attributes"` - Syslog *LogSyslogCfg `yaml:"syslog"` - Tcp *LogTcpCfg `yaml:"tcp"` - Fluentbit *LogExternalFBCfg `yaml:"fluentbit"` - Winlog *LogWinlogCfg `yaml:"winlog"` - Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"` - targetFilesCnt int + Name string `yaml:"name"` + File string `yaml:"file"` // ... + MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines. + Systemd string `yaml:"systemd"` // ... + Pattern string `yaml:"pattern"` + Attributes map[string]string `yaml:"attributes"` + Syslog *LogSyslogCfg `yaml:"syslog"` + Tcp *LogTcpCfg `yaml:"tcp"` + Fluentbit *LogExternalFBCfg `yaml:"fluentbit"` + Winlog *LogWinlogCfg `yaml:"winlog"` + Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"` + MultilineParser string `yaml:"multilineParser"` + targetFilesCnt int } // LogSyslogCfg logging integration config from customer defined YAML, specific for the Syslog input plugin @@ -187,6 +188,7 @@ type FBCfgInput struct { BufferMaxSize string // plugin: tail MemBufferLimit string // plugin: tail PathKey string // plugin: tail + MultilineParser string // plugin: tail SkipLongLines string // always on Systemd_Filter string // plugin: systemd Channels string // plugin: winlog @@ -389,7 +391,7 @@ func parseConfigBlock(l LogCfg, logsHomeDir string, fbOSConfig FBOSConfig) (inpu // Single file func parseFileInput(l LogCfg, dbPath string) (input FBCfgInput, filters []FBCfgFilter) { - input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l)) + input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l), l.MultilineParser) filters = append(filters, newRecordModifierFilterForInput(l.Name, fbInputTypeTail, l.Attributes)) filters = parsePattern(l, fbGrepFieldForTail, filters) return input, filters @@ -544,16 +546,17 @@ func newFBExternalConfig(l LogExternalFBCfg) FBCfgExternal { } } -func newFileInput(filePath string, dbPath string, tag string, bufSize int) FBCfgInput { +func newFileInput(filePath string, dbPath string, tag string, bufSize int, multilineParser string) FBCfgInput { return FBCfgInput{ - Name: fbInputTypeTail, - PathKey: "filePath", - Path: filePath, - DB: dbPath, - Tag: tag, - BufferMaxSize: fmt.Sprintf("%dk", bufSize), - MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit), - SkipLongLines: "On", + Name: fbInputTypeTail, + PathKey: "filePath", + Path: filePath, + DB: dbPath, + Tag: tag, + BufferMaxSize: fmt.Sprintf("%dk", bufSize), + MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit), + MultilineParser: multilineParser, + SkipLongLines: "On", } } diff --git a/pkg/integrations/v4/logs/cfg_template.go b/pkg/integrations/v4/logs/cfg_template.go index 3216de72d..86a3cede3 100644 --- a/pkg/integrations/v4/logs/cfg_template.go +++ b/pkg/integrations/v4/logs/cfg_template.go @@ -20,6 +20,9 @@ var fbConfigFormat = `{{- range .Inputs }} {{- if .SkipLongLines }} Skip_Long_Lines {{ .SkipLongLines }} {{- end }} + {{- if .MultilineParser }} + Multiline.Parser {{ .MultilineParser }} + {{- end }} {{- if .PathKey }} Path_Key {{ .PathKey }} {{- end }} diff --git a/pkg/integrations/v4/logs/cfg_test.go b/pkg/integrations/v4/logs/cfg_test.go index e720d4180..73bd4214c 100644 --- a/pkg/integrations/v4/logs/cfg_test.go +++ b/pkg/integrations/v4/logs/cfg_test.go @@ -1929,3 +1929,193 @@ func TestNewFbConfForTargetFileCount(t *testing.T) { }) } } + +func TestMultilineParserFBCfgFormat(t *testing.T) { + expected := ` +[INPUT] + Name tail + Path /path/to/folder/* + Buffer_Max_Size 32k + Skip_Long_Lines On + Multiline.Parser go + Path_Key filePath + Tag some-folder + DB fb.db + +[FILTER] + Name grep + Match some-folder + Regex log foo + +[FILTER] + Name record_modifier + Match some-folder + Record "fb.input" "tail" + Record "key1" "value1" + Record "key2" "value2" + Record "key3 with space" "value3 with space" + +[FILTER] + Name record_modifier + Match * + Record "entity.guid.INFRA" "testGUID" + Record "fb.source" "nri-agent" + +[OUTPUT] + Name newrelic + Match * + licenseKey ${NR_LICENSE_KEY_ENV_VAR} + validateProxyCerts false + +@INCLUDE /path/to/fb/config +` + + fbCfg := FBCfg{ + Inputs: []FBCfgInput{ + { + Name: "tail", + Tag: "some-folder", + DB: "fb.db", + Path: "/path/to/folder/*", + BufferMaxSize: "32k", + SkipLongLines: "On", + MultilineParser: "go", + PathKey: "filePath", + }, + }, + Filters: []FBCfgFilter{ + { + Name: "grep", + Match: "some-folder", + Regex: "log foo", + }, + { + Name: "record_modifier", + Match: "some-folder", + Records: map[string]string{ + "fb.input": "tail", + "key1": "value1", + "key2": "value2", + "key3 with space": "value3 with space", + }, + }, + { + Name: "record_modifier", + Match: "*", + Records: map[string]string{ + "entity.guid.INFRA": "testGUID", + "fb.source": "nri-agent", + }, + }, + }, + Output: FBCfgOutput{ + Name: "newrelic", + Match: "*", + LicenseKey: "licenseKey", + }, + ExternalCfg: FBCfgExternal{ + CfgFilePath: "/path/to/fb/config", + ParsersFilePath: "/path/to/fb/parsers", + }, + } + + result, extCfg, err := fbCfg.Format() + assert.Empty(t, err) + assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath) + assert.Equal(t, expected, result) +} + +func TestMlParserFBCfgWithMultipleParsers(t *testing.T) { + expected := ` +[INPUT] + Name tail + Path /path/to/folder/* + Buffer_Max_Size 32k + Skip_Long_Lines On + Multiline.Parser go, java, python + Path_Key filePath + Tag some-folder + DB fb.db + +[FILTER] + Name grep + Match some-folder + Regex log foo + +[FILTER] + Name record_modifier + Match some-folder + Record "fb.input" "tail" + Record "key1" "value1" + Record "key2" "value2" + Record "key3 with space" "value3 with space" + +[FILTER] + Name record_modifier + Match * + Record "entity.guid.INFRA" "testGUID" + Record "fb.source" "nri-agent" + +[OUTPUT] + Name newrelic + Match * + licenseKey ${NR_LICENSE_KEY_ENV_VAR} + validateProxyCerts false + +@INCLUDE /path/to/fb/config +` + + fbCfg := FBCfg{ + Inputs: []FBCfgInput{ + { + Name: "tail", + Tag: "some-folder", + DB: "fb.db", + Path: "/path/to/folder/*", + BufferMaxSize: "32k", + SkipLongLines: "On", + MultilineParser: "go, java, python", + PathKey: "filePath", + }, + }, + Filters: []FBCfgFilter{ + { + Name: "grep", + Match: "some-folder", + Regex: "log foo", + }, + { + Name: "record_modifier", + Match: "some-folder", + Records: map[string]string{ + "fb.input": "tail", + "key1": "value1", + "key2": "value2", + "key3 with space": "value3 with space", + }, + }, + { + Name: "record_modifier", + Match: "*", + Records: map[string]string{ + "entity.guid.INFRA": "testGUID", + "fb.source": "nri-agent", + }, + }, + }, + Output: FBCfgOutput{ + Name: "newrelic", + Match: "*", + LicenseKey: "licenseKey", + }, + ExternalCfg: FBCfgExternal{ + CfgFilePath: "/path/to/fb/config", + ParsersFilePath: "/path/to/fb/parsers", + }, + } + + result, extCfg, err := fbCfg.Format() + assert.Empty(t, err) + assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath) + assert.Equal(t, expected, result) +}