Add Elasticsearch templates that work with 2.x

CHANGELOG.asciidoc

@@ -71,6 +71,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha1...master[Check the HEAD d
 - Configuration options and CLI flags for setting the home, data and config paths. {pull}1373[1373]
 - Configuration options and CLI flags for setting the default logs path. {pull}1437[1437]
 - Update to Go 1.6.2 {pull}1447[1447]
+- Add Elasticsearch template files compatible with Elasticsearch 2.x. {pull}1501[1501]
 
 *Packetbeat*
 

dev-tools/packer/platforms/binary/run.sh.j2

@@ -15,6 +15,7 @@ cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-{{.bin_arch}}/
 cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-{{.bin_arch}}/{{.beat_name}}
 cp {{.beat_name}}-binary.yml /{{.beat_name}}-${VERSION}-{{.bin_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-{{.bin_arch}}/
+cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-{{.bin_arch}}/
 
 mkdir -p upload/{{.beat_name}}
 tar czvf upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-{{.bin_arch}}

dev-tools/packer/platforms/centos/run.sh.j2

@@ -36,6 +36,7 @@ fpm --force -s dir -t rpm \
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
         {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
+        {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
 

dev-tools/packer/platforms/darwin/run.sh.j2

@@ -15,6 +15,7 @@ cp -a homedirs/{{.beat_name}}/. /{{.beat_name}}-${VERSION}-darwin/
 cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin/{{.beat_name}}
 cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin/{{.beat_name}}.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin/
+cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin/
 
 mkdir -p upload/{{.beat_name}}
 tar czvf upload/{{.beat_name}}/{{.beat_name}}-${VERSION}-darwin.tgz /{{.beat_name}}-${VERSION}-darwin

dev-tools/packer/platforms/debian/run.sh.j2

@@ -36,6 +36,7 @@ fpm --force -s dir -t deb \
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
         {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
+        {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
 

dev-tools/packer/platforms/windows/run.sh.j2

@@ -16,6 +16,7 @@ cp {{.beat_name}}-windows-{{.arch}}.exe /{{.beat_name}}-${VERSION}-windows/{{.be
 unix2dos {{.beat_name}}-win.yml
 cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows/{{.beat_name}}.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows/
+cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows/
 cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows/
 cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows/
 

dev-tools/packer/xgo-scripts/before_build.sh

@@ -29,6 +29,7 @@ fi
 
 # Copy template
 cp $BEATNAME.template.json $PREFIX/$BEATNAME.template.json
+cp $BEATNAME.template-es2x.json $PREFIX/$BEATNAME.template-es2x.json
 
 # linux
 cp $BEATNAME.yml $PREFIX/$BEATNAME-linux.yml

filebeat/filebeat.template-es2x.json

@@ -0,0 +1,73 @@
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "norms": {
+          "enabled": false
+        }
+      },
+      "dynamic_templates": [
+        {
+          "fields": {
+            "mapping": {
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "string"
+            },
+            "match_mapping_type": "string",
+            "path_match": "fields.*"
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "beat": {
+          "properties": {
+            "hostname": {
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "string"
+            },
+            "name": {
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "string"
+            }
+          }
+        },
+        "input_type": {
+          "ignore_above": 1024,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "message": {
+          "index": "analyzed",
+          "norms": {
+            "enabled": false
+          },
+          "type": "string"
+        },
+        "offset": {
+          "type": "long"
+        },
+        "source": {
+          "ignore_above": 1024,
+          "index": "not_analyzed",
+          "type": "string"
+        },
+        "type": {
+          "ignore_above": 1024,
+          "index": "not_analyzed",
+          "type": "string"
+        }
+      }
+    }
+  },
+  "order": 0,
+  "settings": {
+    "index.refresh_interval": "5s"
+  },
+  "template": "filebeat-*"
+}

filebeat/filebeat.template.json

@@ -9,7 +9,6 @@
           "fields": {
             "mapping": {
               "ignore_above": 1024,
-              "index": true,
               "type": "keyword"
             },
             "match_mapping_type": "string",

libbeat/scripts/generate_template.py

@@ -9,12 +9,12 @@
    python generate_template.py filebeat/ filebeat
 """
 
-import sys
 import yaml
 import json
+import argparse
 
 
-def fields_to_es_template(input, output, index):
+def fields_to_es_template(args, input, output, index):
     """
     Reads the YAML file from input and generates the JSON for
     the ES template in output. input and output are both file
@@ -63,11 +63,18 @@ def fields_to_es_template(input, output, index):
         }
     }
 
+    if args.es2x:
+        # different syntax for norms
+        template["mappings"]["_default_"]["_all"]["norms"] = {
+            "enabled": False
+        }
+
     properties = {}
     dynamic_templates = []
     for doc, section in docs.items():
         if doc != "defaults":
-            prop, dynamic = fill_section_properties(section, defaults, "")
+            prop, dynamic = fill_section_properties(args, section,
+                                                    defaults, "")
             properties.update(prop)
             dynamic_templates.extend(dynamic)
 
@@ -120,7 +127,7 @@ def dedot(group):
     return group
 
 
-def fill_section_properties(section, defaults, path):
+def fill_section_properties(args, section, defaults, path):
     """
     Traverse the sections tree and fill in the properties
     map.
@@ -129,14 +136,14 @@ def fill_section_properties(section, defaults, path):
     dynamic_templates = []
 
     for field in section["fields"]:
-        prop, dynamic = fill_field_properties(field, defaults, path)
+        prop, dynamic = fill_field_properties(args, field, defaults, path)
         properties.update(prop)
         dynamic_templates.extend(dynamic)
 
     return properties, dynamic_templates
 
 
-def fill_field_properties(field, defaults, path):
+def fill_field_properties(args, field, defaults, path):
     """
     Add data about a particular field in the properties
     map.
@@ -148,21 +155,39 @@ def fill_field_properties(field, defaults, path):
         if key not in field:
             field[key] = defaults[key]
 
-    # TODO: Make this more dyanmic
     if field["type"] == "text":
-        properties[field["name"]] = {
-            "type": field["type"],
-            "norms": False
-        }
+        if args.es2x:
+            properties[field["name"]] = {
+                "type": "string",
+                "index": "analyzed",
+                "norms": {
+                    "enabled": False
+                }
+            }
+        else:
+            properties[field["name"]] = {
+                "type": field["type"],
+                "norms": False
+            }
+
+    elif field["type"] == "keyword":
+        if args.es2x:
+            properties[field["name"]] = {
+                "type": "string",
+                "index": "not_analyzed",
+                "ignore_above": 1024
+            }
+        else:
+            properties[field["name"]] = {
+                "type": "keyword",
+                "ignore_above": 1024
+            }
 
     elif field["type"] in ["geo_point", "date", "long", "integer",
-                           "double", "float", "keyword", "boolean"]:
+                           "double", "float", "boolean"]:
         properties[field["name"]] = {
             "type": field.get("type")
         }
-        if field["type"] == "keyword":
-            properties[field["name"]]["ignore_above"] = \
-                defaults.get("ignore_above", 1024)
 
     elif field["type"] in ["dict", "list"]:
         if field.get("dict-type") == "keyword":
@@ -173,24 +198,36 @@ def fill_field_properties(field, defaults, path):
             else:
                 name = field["name"]
 
-            dynamic_templates.append({
-                name: {
-                    "mapping": {
-                        "index": True,
-                        "type": "keyword",
-                        "ignore_above": 1024
-                    },
-                    "match_mapping_type": "string",
-                    "path_match": name + ".*"
-                }
-            })
+            if args.es2x:
+                dynamic_templates.append({
+                    name: {
+                        "mapping": {
+                            "type": "string",
+                            "index": "not_analyzed",
+                            "ignore_above": 1024
+                        },
+                        "match_mapping_type": "string",
+                        "path_match": name + ".*"
+                    }
+                })
+            else:
+                dynamic_templates.append({
+                    name: {
+                        "mapping": {
+                            "type": "keyword",
+                            "ignore_above": 1024
+                        },
+                        "match_mapping_type": "string",
+                        "path_match": name + ".*"
+                    }
+                })
 
     elif field.get("type") == "group":
         if len(path) > 0:
             path = path + "." + field["name"]
         else:
             path = field["name"]
-        prop, dynamic = fill_section_properties(field, defaults, path)
+        prop, dynamic = fill_section_properties(args, field, defaults, path)
 
         # Only add properties if they have a content
         if len(prop) is not 0:
@@ -207,7 +244,10 @@ def fill_field_properties(field, defaults, path):
 
         # Only add properties if they have a content
         if len(prop) is not 0:
-            properties[field.get("name")] = {"type": "nested", "properties": {}}
+            properties[field.get("name")] = {
+                "type": "nested",
+                "properties": {}
+            }
             properties[field.get("name")]["properties"] = prop
 
         dynamic_templates.extend(dynamic)
@@ -219,18 +259,19 @@ def fill_field_properties(field, defaults, path):
 
 if __name__ == "__main__":
 
-    if len(sys.argv) != 3:
-        print "Usage: %s beatpath beatname" % sys.argv[0]
-        sys.exit(1)
-
-    beat_path = sys.argv[1]
-    beat_name = sys.argv[2]
-
-    input = open(beat_path + "/etc/fields.yml", 'r')
-    output = open(beat_path + "/" + beat_name + ".template.json", 'w')
-
-    try:
-        fields_to_es_template(input, output, beat_name + "-*")
-    finally:
-        input.close()
-        output.close()
+    parser = argparse.ArgumentParser(
+        description="Generates the templates for a Beat.")
+    parser.add_argument("--es2x", action="store_true",
+                        help="Generate template for Elasticsearch 2.x.")
+    parser.add_argument("path", help="Path to the beat folder")
+    parser.add_argument("beatname", help="The beat fname")
+    args = parser.parse_args()
+
+    target = args.path + "/" + args.beatname + ".template"
+    if args.es2x:
+        target += "-es2x"
+    target += ".json"
+
+    with open(args.path + "/etc/fields.yml", 'r') as input:
+        with open(target, 'w') as output:
+            fields_to_es_template(args, input, output, args.beatname + "-*")

libbeat/scripts/update.sh

@@ -43,3 +43,4 @@ python ${LIBBEAT}/scripts/generate_fields_docs.py $(pwd) ${BEATNAME}
 
 # Update templates
 python ${LIBBEAT}/scripts/generate_template.py $(pwd) ${BEATNAME}
+python ${LIBBEAT}/scripts/generate_template.py --es2x $(pwd) ${BEATNAME}