Ensure doc_values parameter is passed into Beats field defintions

CHANGELOG.next.md

@@ -57,6 +57,7 @@ Thanks, you're awesome :-) -->
 #### Bugfixes
 
 * Remove `ignore_above` when `index: false` and `doc_values: false`. #1483
+* Ensure `doc_values` is carried into Beats artifacts. #1488
 
 #### Added
 

experimental/generated/beats/fields.ecs.yml

@@ -1935,6 +1935,7 @@
       example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|
         worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232
       index: false
+      doc_values: false
     - name: outcome
       level: core
       type: keyword
@@ -2849,6 +2850,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: x509.public_key_size
       level: extended
@@ -3566,6 +3568,7 @@
         but the value can be retrieved from `_source`.'
       example: Sep 19 08:26:10 localhost My log
       index: false
+      doc_values: false
     - name: syslog
       level: extended
       type: object
@@ -8384,6 +8387,7 @@
       example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|
         worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232
       index: false
+      doc_values: false
       default_field: false
     - name: enrichments.event.outcome
       level: core
@@ -9643,6 +9647,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: enrichments.x509.public_key_size
       level: extended
@@ -10185,6 +10190,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: client.x509.public_key_size
       level: extended
@@ -10462,6 +10468,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: server.x509.public_key_size
       level: extended
@@ -11400,6 +11407,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: public_key_size
       level: extended

generated/beats/fields.ecs.yml

@@ -1547,6 +1547,7 @@
         default_field: false
       description: The stack trace of this error in plain text.
       index: false
+      doc_values: false
     - name: type
       level: extended
       type: keyword
@@ -1746,6 +1747,7 @@
       example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|
         worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232
       index: false
+      doc_values: false
     - name: outcome
       level: core
       type: keyword
@@ -2455,6 +2457,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: x509.public_key_size
       level: extended
@@ -3183,6 +3186,7 @@
         but the value can be retrieved from `_source`.'
       example: Sep 19 08:26:10 localhost My log
       index: false
+      doc_values: false
     - name: syslog
       level: extended
       type: object
@@ -6052,6 +6056,7 @@
       example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100|
         worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232
       index: false
+      doc_values: false
       default_field: false
     - name: enrichments.event.outcome
       level: core
@@ -7002,6 +7007,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: enrichments.x509.public_key_size
       level: extended
@@ -7548,6 +7554,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: client.x509.public_key_size
       level: extended
@@ -7829,6 +7836,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: server.x509.public_key_size
       level: extended
@@ -8789,6 +8797,7 @@
       description: Exponent used to derive the public key. This is algorithm specific.
       example: 65537
       index: false
+      doc_values: false
       default_field: false
     - name: public_key_size
       level: extended

scripts/generators/beats.py

@@ -39,7 +39,8 @@ def fieldset_field_array(source_fields, df_allowlist, fieldset_prefix):
     allowed_keys = ['name', 'level', 'required', 'type', 'object_type',
                     'ignore_above', 'multi_fields', 'format', 'input_format',
                     'output_format', 'output_precision', 'description',
-                    'example', 'enabled', 'index', 'path', 'scaling_factor']
+                    'example', 'enabled', 'index', 'doc_values', 'path',
+                    'scaling_factor']
     multi_fields_allowed_keys = ['name', 'type', 'norms', 'default_field', 'normalizer', 'ignore_above']
 
     fields = []

scripts/tests/unit/test_beats_generator.py

@@ -0,0 +1,78 @@
+import os
+import sys
+from typing import OrderedDict
+import unittest
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+from scripts.generators import beats
+
+
+class TestGeneratorsBeatsFields(unittest.TestCase):
+    def setUp(self):
+        self.df_allowlist = {"@timestamp"}
+
+    def test_fieldset_field_array_expected_structure(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'ignore_above': 1024,
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword'
+            }
+        }
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, 'prefix')
+        self.assertIsInstance(beats_fields, list)
+        self.assertIsInstance(beats_fields[0], OrderedDict)
+
+    def test_fieldset_field_array_expected_values_keyword(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'ignore_above': 1024,
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword'
+            }
+        }
+
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '')
+        field_entry = beats_fields[0]
+        self.assertEqual(field_entry['type'], 'keyword')
+        self.assertEqual(field_entry['ignore_above'], 1024)
+        self.assertEqual(field_entry['name'], 'id')
+
+    def test_fieldset_field_array_expected_values_keyword_index_false(self):
+        fields = {
+            'id': {
+                'dashed_name': 'agent-id',
+                'description': 'description',
+                'flat_name': 'agent.id',
+                'level': 'extended',
+                'name': 'agent.id',
+                'normalize': [],
+                'short': 'short',
+                'type': 'keyword',
+                'index': False,
+                'doc_values': False
+            }
+        }
+
+        beats_fields = beats.fieldset_field_array(fields, self.df_allowlist, '')
+        field_entry = beats_fields[0]
+        self.assertEqual(field_entry['type'], 'keyword')
+        self.assertFalse(field_entry['index'])
+        self.assertFalse(field_entry['doc_values'])
+
+
+if __name__ == '__main__':
+    unittest.main()