diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 4713f27bf4..a051b25ad4 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -57,6 +57,7 @@ Thanks, you're awesome :-) --> #### Bugfixes * Remove `ignore_above` when `index: false` and `doc_values: false`. #1483 +* Ensure `doc_values` is carried into Beats artifacts. #1488 #### Added diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index e6ccca4647..9a9c890735 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1935,6 +1935,7 @@ example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 index: false + doc_values: false - name: outcome level: core type: keyword @@ -2849,6 +2850,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: x509.public_key_size level: extended @@ -3566,6 +3568,7 @@ but the value can be retrieved from `_source`.' example: Sep 19 08:26:10 localhost My log index: false + doc_values: false - name: syslog level: extended type: object @@ -8384,6 +8387,7 @@ example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 index: false + doc_values: false default_field: false - name: enrichments.event.outcome level: core @@ -9643,6 +9647,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: enrichments.x509.public_key_size level: extended @@ -10185,6 +10190,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: client.x509.public_key_size level: extended @@ -10462,6 +10468,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: server.x509.public_key_size level: extended @@ -11400,6 +11407,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: public_key_size level: extended diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index fcccfb404b..2854637315 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1547,6 +1547,7 @@ default_field: false description: The stack trace of this error in plain text. index: false + doc_values: false - name: type level: extended type: keyword @@ -1746,6 +1747,7 @@ example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 index: false + doc_values: false - name: outcome level: core type: keyword @@ -2455,6 +2457,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: x509.public_key_size level: extended @@ -3183,6 +3186,7 @@ but the value can be retrieved from `_source`.' example: Sep 19 08:26:10 localhost My log index: false + doc_values: false - name: syslog level: extended type: object @@ -6052,6 +6056,7 @@ example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 index: false + doc_values: false default_field: false - name: enrichments.event.outcome level: core @@ -7002,6 +7007,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: enrichments.x509.public_key_size level: extended @@ -7548,6 +7554,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: client.x509.public_key_size level: extended @@ -7829,6 +7836,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: server.x509.public_key_size level: extended @@ -8789,6 +8797,7 @@ description: Exponent used to derive the public key. This is algorithm specific. example: 65537 index: false + doc_values: false default_field: false - name: public_key_size level: extended diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 5708a03555..f36eebad55 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -39,7 +39,8 @@ def fieldset_field_array(source_fields, df_allowlist, fieldset_prefix): allowed_keys = ['name', 'level', 'required', 'type', 'object_type', 'ignore_above', 'multi_fields', 'format', 'input_format', 'output_format', 'output_precision', 'description', - 'example', 'enabled', 'index', 'path', 'scaling_factor'] + 'example', 'enabled', 'index', 'doc_values', 'path', + 'scaling_factor'] multi_fields_allowed_keys = ['name', 'type', 'norms', 'default_field', 'normalizer', 'ignore_above'] fields = [] diff --git a/scripts/tests/unit/test_beats_generator.py b/scripts/tests/unit/test_beats_generator.py new file mode 100644 index 0000000000..a433442557 --- /dev/null +++ b/scripts/tests/unit/test_beats_generator.py @@ -0,0 +1,78 @@ +import os +import sys +from typing import OrderedDict +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../..')) + +from scripts.generators import beats + + +class TestGeneratorsBeatsFields(unittest.TestCase): + def setUp(self): + self.df_allowlist = {"@timestamp"} + + def test_fieldset_field_array_expected_structure(self): + fields = { + 'id': { + 'dashed_name': 'agent-id', + 'description': 'description', + 'flat_name': 'agent.id', + 'ignore_above': 1024, + 'level': 'extended', + 'name': 'agent.id', + 'normalize': [], + 'short': 'short', + 'type': 'keyword' + } + } + beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, 'prefix') + self.assertIsInstance(beats_fields, list) + self.assertIsInstance(beats_fields[0], OrderedDict) + + def test_fieldset_field_array_expected_values_keyword(self): + fields = { + 'id': { + 'dashed_name': 'agent-id', + 'description': 'description', + 'flat_name': 'agent.id', + 'ignore_above': 1024, + 'level': 'extended', + 'name': 'agent.id', + 'normalize': [], + 'short': 'short', + 'type': 'keyword' + } + } + + beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '') + field_entry = beats_fields[0] + self.assertEqual(field_entry['type'], 'keyword') + self.assertEqual(field_entry['ignore_above'], 1024) + self.assertEqual(field_entry['name'], 'id') + + def test_fieldset_field_array_expected_values_keyword_index_false(self): + fields = { + 'id': { + 'dashed_name': 'agent-id', + 'description': 'description', + 'flat_name': 'agent.id', + 'level': 'extended', + 'name': 'agent.id', + 'normalize': [], + 'short': 'short', + 'type': 'keyword', + 'index': False, + 'doc_values': False + } + } + + beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '') + field_entry = beats_fields[0] + self.assertEqual(field_entry['type'], 'keyword') + self.assertFalse(field_entry['index']) + self.assertFalse(field_entry['doc_values']) + + +if __name__ == '__main__': + unittest.main()