Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NR-279356: Add new multiline parser config param to logging config #1881

Merged
merged 1 commit into from
Jun 14, 2024
Merged

NR-279356: Add new multiline parser config param to logging config #1881

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 25 additions & 22 deletions pkg/integrations/v4/logs/cfg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,19 @@ type YAML struct {

// LogCfg logging integration config from customer defined YAML.
type LogCfg struct {
Name string `yaml:"name"`
File string `yaml:"file"` // ...
MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines.
Systemd string `yaml:"systemd"` // ...
Pattern string `yaml:"pattern"`
Attributes map[string]string `yaml:"attributes"`
Syslog *LogSyslogCfg `yaml:"syslog"`
Tcp *LogTcpCfg `yaml:"tcp"`
Fluentbit *LogExternalFBCfg `yaml:"fluentbit"`
Winlog *LogWinlogCfg `yaml:"winlog"`
Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"`
targetFilesCnt int
Name string `yaml:"name"`
File string `yaml:"file"` // ...
MaxLineKb int `yaml:"max_line_kb"` // Setup the max value of the buffer while reading lines.
Systemd string `yaml:"systemd"` // ...
Pattern string `yaml:"pattern"`
Attributes map[string]string `yaml:"attributes"`
Syslog *LogSyslogCfg `yaml:"syslog"`
Tcp *LogTcpCfg `yaml:"tcp"`
Fluentbit *LogExternalFBCfg `yaml:"fluentbit"`
Winlog *LogWinlogCfg `yaml:"winlog"`
Winevtlog *LogWinevtlogCfg `yaml:"winevtlog"`
MultilineParser string `yaml:"multilineParser"`
targetFilesCnt int
}

// LogSyslogCfg logging integration config from customer defined YAML, specific for the Syslog input plugin
Expand Down Expand Up @@ -187,6 +188,7 @@ type FBCfgInput struct {
BufferMaxSize string // plugin: tail
MemBufferLimit string // plugin: tail
PathKey string // plugin: tail
MultilineParser string // plugin: tail
SkipLongLines string // always on
Systemd_Filter string // plugin: systemd
Channels string // plugin: winlog
Expand Down Expand Up @@ -389,7 +391,7 @@ func parseConfigBlock(l LogCfg, logsHomeDir string, fbOSConfig FBOSConfig) (inpu

// Single file
func parseFileInput(l LogCfg, dbPath string) (input FBCfgInput, filters []FBCfgFilter) {
input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l))
input = newFileInput(l.File, dbPath, l.Name, getBufferMaxSize(l), l.MultilineParser)
filters = append(filters, newRecordModifierFilterForInput(l.Name, fbInputTypeTail, l.Attributes))
filters = parsePattern(l, fbGrepFieldForTail, filters)
return input, filters
Expand Down Expand Up @@ -544,16 +546,17 @@ func newFBExternalConfig(l LogExternalFBCfg) FBCfgExternal {
}
}

func newFileInput(filePath string, dbPath string, tag string, bufSize int) FBCfgInput {
func newFileInput(filePath string, dbPath string, tag string, bufSize int, multilineParser string) FBCfgInput {
return FBCfgInput{
Name: fbInputTypeTail,
PathKey: "filePath",
Path: filePath,
DB: dbPath,
Tag: tag,
BufferMaxSize: fmt.Sprintf("%dk", bufSize),
MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit),
SkipLongLines: "On",
Name: fbInputTypeTail,
PathKey: "filePath",
Path: filePath,
DB: dbPath,
Tag: tag,
BufferMaxSize: fmt.Sprintf("%dk", bufSize),
MemBufferLimit: fmt.Sprintf("%dk", memBufferLimit),
MultilineParser: multilineParser,
SkipLongLines: "On",
}
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/integrations/v4/logs/cfg_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ var fbConfigFormat = `{{- range .Inputs }}
{{- if .SkipLongLines }}
Skip_Long_Lines {{ .SkipLongLines }}
{{- end }}
{{- if .MultilineParser }}
Multiline.Parser {{ .MultilineParser }}
{{- end }}
{{- if .PathKey }}
Path_Key {{ .PathKey }}
{{- end }}
Expand Down
190 changes: 190 additions & 0 deletions pkg/integrations/v4/logs/cfg_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1929,3 +1929,193 @@ func TestNewFbConfForTargetFileCount(t *testing.T) {
})
}
}

func TestMultilineParserFBCfgFormat(t *testing.T) {
expected := `
[INPUT]
Name tail
Path /path/to/folder/*
Buffer_Max_Size 32k
Skip_Long_Lines On
Multiline.Parser go
Path_Key filePath
Tag some-folder
DB fb.db

[FILTER]
Name grep
Match some-folder
Regex log foo

[FILTER]
Name record_modifier
Match some-folder
Record "fb.input" "tail"
Record "key1" "value1"
Record "key2" "value2"
Record "key3 with space" "value3 with space"

[FILTER]
Name record_modifier
Match *
Record "entity.guid.INFRA" "testGUID"
Record "fb.source" "nri-agent"

[OUTPUT]
Name newrelic
Match *
licenseKey ${NR_LICENSE_KEY_ENV_VAR}
validateProxyCerts false

@INCLUDE /path/to/fb/config
`

fbCfg := FBCfg{
Inputs: []FBCfgInput{
{
Name: "tail",
Tag: "some-folder",
DB: "fb.db",
Path: "/path/to/folder/*",
BufferMaxSize: "32k",
SkipLongLines: "On",
MultilineParser: "go",
PathKey: "filePath",
},
},
Filters: []FBCfgFilter{
{
Name: "grep",
Match: "some-folder",
Regex: "log foo",
},
{
Name: "record_modifier",
Match: "some-folder",
Records: map[string]string{
"fb.input": "tail",
"key1": "value1",
"key2": "value2",
"key3 with space": "value3 with space",
},
},
{
Name: "record_modifier",
Match: "*",
Records: map[string]string{
"entity.guid.INFRA": "testGUID",
"fb.source": "nri-agent",
},
},
},
Output: FBCfgOutput{
Name: "newrelic",
Match: "*",
LicenseKey: "licenseKey",
},
ExternalCfg: FBCfgExternal{
CfgFilePath: "/path/to/fb/config",
ParsersFilePath: "/path/to/fb/parsers",
},
}

result, extCfg, err := fbCfg.Format()
assert.Empty(t, err)
assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath)
assert.Equal(t, expected, result)
}

func TestMlParserFBCfgWithMultipleParsers(t *testing.T) {
expected := `
[INPUT]
Name tail
Path /path/to/folder/*
Buffer_Max_Size 32k
Skip_Long_Lines On
Multiline.Parser go, java, python
Path_Key filePath
Tag some-folder
DB fb.db

[FILTER]
Name grep
Match some-folder
Regex log foo

[FILTER]
Name record_modifier
Match some-folder
Record "fb.input" "tail"
Record "key1" "value1"
Record "key2" "value2"
Record "key3 with space" "value3 with space"

[FILTER]
Name record_modifier
Match *
Record "entity.guid.INFRA" "testGUID"
Record "fb.source" "nri-agent"

[OUTPUT]
Name newrelic
Match *
licenseKey ${NR_LICENSE_KEY_ENV_VAR}
validateProxyCerts false

@INCLUDE /path/to/fb/config
`

fbCfg := FBCfg{
Inputs: []FBCfgInput{
{
Name: "tail",
Tag: "some-folder",
DB: "fb.db",
Path: "/path/to/folder/*",
BufferMaxSize: "32k",
SkipLongLines: "On",
MultilineParser: "go, java, python",
PathKey: "filePath",
},
},
Filters: []FBCfgFilter{
{
Name: "grep",
Match: "some-folder",
Regex: "log foo",
},
{
Name: "record_modifier",
Match: "some-folder",
Records: map[string]string{
"fb.input": "tail",
"key1": "value1",
"key2": "value2",
"key3 with space": "value3 with space",
},
},
{
Name: "record_modifier",
Match: "*",
Records: map[string]string{
"entity.guid.INFRA": "testGUID",
"fb.source": "nri-agent",
},
},
},
Output: FBCfgOutput{
Name: "newrelic",
Match: "*",
LicenseKey: "licenseKey",
},
ExternalCfg: FBCfgExternal{
CfgFilePath: "/path/to/fb/config",
ParsersFilePath: "/path/to/fb/parsers",
},
}

result, extCfg, err := fbCfg.Format()
assert.Empty(t, err)
assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath)
assert.Equal(t, expected, result)
}