From 166023d1340ee38fe222fa5d2c3f8560b0890427 Mon Sep 17 00:00:00 2001 From: Kalman Meth Date: Mon, 1 Aug 2022 16:47:54 +0300 Subject: [PATCH 1/2] support remove_entry_if_equal and remove_entry_if_not_equal --- README.md | 2 + pkg/api/transform_filter.go | 7 +- pkg/pipeline/transform/transform_filter.go | 12 ++++ .../transform/transform_filter_test.go | 71 +++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 57a6cb77b..43451c77b 100644 --- a/README.md +++ b/README.md @@ -361,6 +361,8 @@ pipeline: Using `remove_entry_if_doesnt_exist` in the rule reverses the logic and will not remove the above example entry Using `remove_field` in the rule `type` instead, results in outputting the entry after removal of only the `SrcPort` key and value +Using `remove_entry_if_equal` will remove the entry if the specified field exists and is equal to the specified value. +Using `remove_entry_if_not_equal` will remove the entry if the specified field exists and is not equal to the specified value. ### Transform Network diff --git a/pkg/api/transform_filter.go b/pkg/api/transform_filter.go index d467474ab..8383a9359 100644 --- a/pkg/api/transform_filter.go +++ b/pkg/api/transform_filter.go @@ -25,6 +25,8 @@ type TransformFilterOperationEnum struct { RemoveField string `yaml:"remove_field" json:"remove_field" doc:"removes the field from the entry"` RemoveEntryIfExists string `yaml:"remove_entry_if_exists" json:"remove_entry_if_exists" doc:"removes the entry if the field exists"` RemoveEntryIfDoesntExist string `yaml:"remove_entry_if_doesnt_exist" json:"remove_entry_if_doesnt_exist" doc:"removes the entry if the field doesnt exist"` + RemoveEntryIfEqual string `yaml:"remove_entry_if_equal" json:"remove_entry_if_equal" doc:"removes the entry if the field value equals specified value"` + RemoveEntryIfNotEqual string `yaml:"remove_entry_if_not_equal" json:"remove_entry_if_not_equal" doc:"removes the entry if the field value does not equal specified value"` } func TransformFilterOperationName(operation string) string { @@ -32,6 +34,7 @@ func TransformFilterOperationName(operation string) string { } type TransformFilterRule struct { - Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` - Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformFilterOperationEnum" doc:"one of the following:"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformFilterOperationEnum" doc:"one of the following:"` + Value interface{} `yaml:"value,omitempty" json:"value,omitempty" enum:"TransformFilterOperationEnum" doc:"specified value of input field:"` } diff --git a/pkg/pipeline/transform/transform_filter.go b/pkg/pipeline/transform/transform_filter.go index bd2783177..5365a08ba 100644 --- a/pkg/pipeline/transform/transform_filter.go +++ b/pkg/pipeline/transform/transform_filter.go @@ -48,6 +48,18 @@ func (f *Filter) Transform(input []config.GenericMap) []config.GenericMap { if _, ok := entry[rule.Input]; !ok { addToOutput = false } + case api.TransformFilterOperationName("RemoveEntryIfEqual"): + if val, ok := entry[rule.Input]; ok { + if val == rule.Value { + addToOutput = false + } + } + case api.TransformFilterOperationName("RemoveEntryIfNotEqual"): + if val, ok := entry[rule.Input]; ok { + if val != rule.Value { + addToOutput = false + } + } default: log.Panicf("unknown type %s for transform.Filter rule: %v", rule.Type, rule) } diff --git a/pkg/pipeline/transform/transform_filter_test.go b/pkg/pipeline/transform/transform_filter_test.go index c5453f1a6..45b581aeb 100644 --- a/pkg/pipeline/transform/transform_filter_test.go +++ b/pkg/pipeline/transform/transform_filter_test.go @@ -68,6 +68,38 @@ parameters: - input: doesntSrcPort type: remove_entry_if_doesnt_exist ` +const testConfigTransformFilterRemoveEntryIfEqual = `--- +log-level: debug +pipeline: + - name: filter1 +parameters: + - name: filter1 + transform: + type: filter + filter: + rules: + - input: message + type: remove_entry_if_equal + value: "test message" + - input: value + type: remove_entry_if_equal + value: 8.0 +` + +const testConfigTransformFilterRemoveEntryIfNotEqual = `--- +log-level: debug +pipeline: + - name: filter1 +parameters: + - name: filter1 + transform: + type: filter + filter: + rules: + - input: message + type: remove_entry_if_not_equal + value: "test message" +` func getFilterExpectedOutput() config.GenericMap { return config.GenericMap{ @@ -113,6 +145,45 @@ func TestNewTransformFilterRemoveEntryIfDoesntExists(t *testing.T) { output := transformFilter.Transform([]config.GenericMap{input}) require.Equal(t, output, []config.GenericMap{}) } + +func TestNewTransformFilterRemoveEntryIfEqual(t *testing.T) { + newTransform := InitNewTransformFilter(t, testConfigTransformFilterRemoveEntryIfEqual) + transformFilter := newTransform.(*Filter) + require.Len(t, transformFilter.Rules, 2) + + input := test.GetIngestMockEntry(false) + + output := transformFilter.Transform([]config.GenericMap{input}) + require.Equal(t, 0, len(output)) + + input["message"] = "dummy message" + output = transformFilter.Transform([]config.GenericMap{input}) + require.Equal(t, 1, len(output)) + require.Contains(t, output[0], "message") + require.Equal(t, output[0]["message"], "dummy message") + + input["value"] = 8.0 + output = transformFilter.Transform([]config.GenericMap{input}) + require.Equal(t, 0, len(output)) +} + +func TestNewTransformFilterRemoveEntryIfNotEqual(t *testing.T) { + newTransform := InitNewTransformFilter(t, testConfigTransformFilterRemoveEntryIfNotEqual) + transformFilter := newTransform.(*Filter) + require.Len(t, transformFilter.Rules, 1) + + input := test.GetIngestMockEntry(false) + + output := transformFilter.Transform([]config.GenericMap{input}) + require.Equal(t, 1, len(output)) + require.Contains(t, output[0], "message") + require.Equal(t, output[0]["message"], "test message") + + input["message"] = "dummy message" + output = transformFilter.Transform([]config.GenericMap{input}) + require.Equal(t, 0, len(output)) +} + func InitNewTransformFilter(t *testing.T, configFile string) Transformer { v, cfg := test.InitConfig(t, configFile) require.NotNil(t, v) From 85bf981b479da04345281436922a2e5cd00c128b Mon Sep 17 00:00:00 2001 From: Kalman Meth Date: Tue, 2 Aug 2022 10:23:01 +0300 Subject: [PATCH 2/2] updated api.md --- docs/api.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/api.md b/docs/api.md index ab674c876..98e5acf9e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -99,6 +99,14 @@ Following is the supported API format for filter transformations: remove_field: removes the field from the entry remove_entry_if_exists: removes the entry if the field exists remove_entry_if_doesnt_exist: removes the entry if the field doesnt exist + remove_entry_if_equal: removes the entry if the field value equals specified value + remove_entry_if_not_equal: removes the entry if the field value does not equal specified value + value: (enum) specified value of input field: + remove_field: removes the field from the entry + remove_entry_if_exists: removes the entry if the field exists + remove_entry_if_doesnt_exist: removes the entry if the field doesnt exist + remove_entry_if_equal: removes the entry if the field value equals specified value + remove_entry_if_not_equal: removes the entry if the field value does not equal specified value ## Transform Network API Following is the supported API format for network transformations: