diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 92fad8bd20..9729d05b03 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -11,6 +11,7 @@ Thanks, you're awesome :-) --> ### Breaking Changes * Removing deprecated --oss from generator #1404 +* Removing use-cases directory #1405 ### Schema Changes diff --git a/Makefile b/Makefile index 327f64b49f..2076cfa82b 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,6 @@ check-license-headers: .PHONY: clean clean: rm -rf build generated/elasticsearch/component experimental/generated/elasticsearch/component - # Clean all markdown files for use-cases - find ./use-cases -type f -name '*.md' -not -name 'README.md' -print0 | xargs -0 rm -- # Alias to generate source code for all languages. .PHONY: codegen @@ -64,7 +62,7 @@ fmt: ve # Alias to generate everything. .PHONY: generate -generate: generator legacy_use_cases codegen +generate: generator codegen $(PYTHON) --version # Run the new generator @@ -82,11 +80,6 @@ gocodegen: -schema=../schemas \ -out=../code/go/ecs -# Generate the Use Cases -.PHONY: legacy_use_cases -legacy_use_cases: ve - $(PYTHON) scripts/use-cases.py --stdout=true >> /dev/null - # Check Makefile format. .PHONY: makelint makelint: SHELL:=/bin/bash diff --git a/docs/use-cases.asciidoc.disabled b/docs/use-cases.asciidoc.disabled deleted file mode 100644 index 687f8dea53..0000000000 --- a/docs/use-cases.asciidoc.disabled +++ /dev/null @@ -1,22 +0,0 @@ -[[ecs-use-cases]] -== Use Cases - -The power and versatility of {ecs} is best illustrated through use cases. - -NOTE: Some use cases contain both ECS fields and additional fields which are not -in ECS to describe the full use case. Non-ECS fields are in italic. - - * https://github.com/elastic/ecs/blob/master/use-cases/apm.md[APM] - * https://github.com/elastic/ecs/blob/master/use-cases/auditbeat.md[Auditbeat] - * https://github.com/elastic/ecs/blob/master/use-cases/beats.md[Beats] - * https://github.com/elastic/ecs/blob/master/use-cases/filebeat-apache-access.md[Filebeat Apache] - * https://github.com/elastic/ecs/blob/master/use-cases/kubernetes.md[Kubernetes] - * https://github.com/elastic/ecs/blob/master/use-cases/logging.md[Logging] - * https://github.com/elastic/ecs/blob/master/use-cases/metricbeat.md[Metricbeat] - * https://github.com/elastic/ecs/blob/master/use-cases/tls.md[TLS] - * https://github.com/elastic/ecs/blob/master/use-cases/web-logs.md[Parsing web server logs] - -We welcome https://github.com/elastic/ecs/blob/master/CONTRIBUTING.md[contributions] of additional ECS uses cases. - - - diff --git a/scripts/helper.py b/scripts/helper.py deleted file mode 100644 index f9bfa05756..0000000000 --- a/scripts/helper.py +++ /dev/null @@ -1,179 +0,0 @@ -import yaml -import glob - - -def read_schema_file(path): - """Read a schema.yml file and cleans up the fields - """ - fields = [] - with open(path) as f: - fields = yaml.safe_load(f.read()) - - clean_namespace_fields(fields) - return fields - - -def read_use_case_file(path): - """Read a use-case.yml file and cleans up the fields - """ - with open(path) as f: - use_case = yaml.safe_load(f.read()) - - fields = use_case["fields"] - clean_namespace_fields(fields) - use_case["fields"] = fields - return use_case - - -def clean_namespace_fields(fields): - """Cleans up all fields to set defaults - """ - for namespace in fields: - - # For now set the default group to 2 - if "group" not in namespace: - namespace["group"] = 2 - - prefix = "" - # Prefix if not base namespace - if namespace["name"] != "base": - prefix = namespace["name"] - - clean_fields(namespace["fields"], prefix, namespace["group"]) - - -def clean_fields(fields, prefix, group): - for field in fields: - clean_string_field(field, "description") - clean_string_field(field, "footnote") - clean_string_field(field, "example") - clean_string_field(field, "type") - - # Add prefix if needed - if prefix != "": - field["name"] = prefix + "." + field["name"] - - if 'level' not in field.keys(): - field["level"] = '(use case)' - - if 'group' not in field.keys(): - # If no group set, set parent group - field["group"] = group - - # if "multi_fields" in field: - # for f in field["multi_fields"]: - # clean_string_field(f, "description") - # clean_string_field(f, "example") - # clean_string_field(f, "type") - - # # multi fields always have a prefix - # f["name"] = field["name"] + "." + f["name"] - - # if 'group' not in f.keys(): - # # If no group set, set parent group - # f["group"] = group - - -def clean_string_field(field, key): - """Cleans a string field and creates an empty string for the field in case it does not exist - """ - if key in field.keys(): - # Remove all spaces and newlines from beginning and end - field[key] = str(field[key]).strip() - else: - field[key] = "" - - if "index" in field and field["index"] == False: - field["type"] = "(not indexed)" - - -def get_markdown_row(field, link, multi_field): - """Creates a markdown table for the given fields - """ - - # Replace newlines with HTML representation as otherwise newlines don't work in Markdown - description = field["description"].replace("\n", "
") - - show_name = field["name"] - - ecs = True - if 'ecs' in field.keys(): - ecs = field["ecs"] - - # non ecs fields are in italic - if not ecs: - show_name = "*" + field["name"] + "*" - description = "*" + description + "*" - - example = "" - if field["example"] != "": - # Add ticks around examples to not break table - example = "`{}`".format(field["example"]) - - # If link is true, it link to the anchor is provided. This is used for the use-cases - if link and ecs: - return '| [{}]({}#{}) | {} | {} | {} | {} |\n'.format(show_name, link, field["name"], description, field["level"], field["type"], example) - - # By default a anchor is attached to the name - return '| {} | {} | {} | {} | {} |\n'.format(field["name"], show_name, description, field["level"], field["type"], example) - - -def get_schema(): - fields = [] - for file in sorted(glob.glob("schemas/*.yml")): - fields = fields + read_schema_file(file) - return fields - - -def get_markdown_section(namespace, title_prefix="##", link=False): - section_name = namespace["name"] - - # Title - output = '{} {} fields\n\n'.format(title_prefix, section_name, namespace["title"]) - - # Description - # Replaces one newlines with two as otherwise double newlines do not show up in markdown - output += namespace["description"].replace("\n", "\n\n") + "\n" - - # Reusable object details - if "reusable" in namespace and "expected" in namespace["reusable"]: - sorted_fields = sorted(namespace["reusable"]["expected"]) - rendered_fields = map(lambda f: "`{}.{}`".format(f, section_name), sorted_fields) - output += "The `{}` fields are expected to be nested at: {}.\n\n".format( - section_name, ', '.join(rendered_fields)) - - if "top_level" in namespace["reusable"] and namespace["reusable"]["top_level"]: - template = "Note also that the `{}` fields may be used directly at the top level.\n\n" - else: - template = "Note also that the `{}` fields are not expected to " + \ - "be used directly at the top level.\n\n" - output += template.format(section_name) - - # Table - titles = ["Field", "Description", "Level", "Type", "Example"] - - for title in titles: - output += "| {} ".format(title) - output += "|\n" - - for title in titles: - output += "|---" - output += "|\n" - - # Sort fields for easier readability - namespaceFields = sorted(namespace["fields"], key=lambda field: field["name"]) - - # Print fields into a table - for field in namespace["fields"]: - output += get_markdown_row(field, link, False) - if "multi_fields" in field: - for f in field["multi_fields"]: - output += get_markdown_row(f, link, True) - - output += "\n\n" - - # Footnote - if "footnote" in namespace: - output += namespace["footnote"].replace("\n", "\n\n") + "\n" - - return output diff --git a/scripts/use-cases.py b/scripts/use-cases.py deleted file mode 100644 index 99df56a8aa..0000000000 --- a/scripts/use-cases.py +++ /dev/null @@ -1,95 +0,0 @@ -import yaml -import os -import argparse -from helper import * -import os.path - - -def write_stdout(): - - schema = get_schema() - flat_schema = create_flat_schema(schema) - - links = "" - for file in sorted(os.listdir("./use-cases")): - - output = "" - - if not file.endswith(".yml"): - continue - - use_case = read_use_case_file("./use-cases/" + file) - - # Intentionally a relative link, to avoid leaving forked repo or branch - schema_link = "use-cases/" - # Link list to field prefixes - links += " * [{}]({}{}.md)\n".format(use_case["title"], schema_link, use_case["name"]) - - output += "## {} use case\n\n".format(use_case["title"]) - output += "{}\n\n".format(use_case["description"]) - - fields = [] - for use_case_section in use_case["fields"]: - # In case a description exists for a prefix, add it as field with .* - if "description" in use_case_section and use_case_section["description"] != "": - fields.append({ - "name": use_case_section["name"] + ".*", - "description": use_case_section["description"], - "type": "", - "level": "", - "example": "", - "ecs": False, - }) - - for section_fields in use_case_section["fields"]: - # Complete ECS fields with ECS information if not set - if section_fields["name"] in flat_schema: - section_fields["ecs"] = True - section_fields["type"] = flat_schema[section_fields["name"]]["type"] - section_fields["level"] = flat_schema[section_fields["name"]]["level"] - if section_fields["description"] == "": - section_fields["description"] = flat_schema[section_fields["name"]]["description"] - if section_fields["example"] == "": - section_fields["example"] = flat_schema[section_fields["name"]]["example"] - else: - section_fields["ecs"] = False - section_fields["level"] = "(use case)" - - fields.append(section_fields) - - global_fields = {"name": use_case["name"], "title": use_case["title"], "description": "", "fields": fields} - # Generate use cases with a relative link to access field definitions - output += get_markdown_section(global_fields, "###", "../README.md") + "\n" - - # Write output to /use-cases/use_case["name"].md file - # Adjust links - - with open("./use-cases/" + use_case["name"] + ".md", "w") as f: - f.write(output) - - print("\n" + links + "\n\n") - - -def create_flat_schema(schema): - fields = {} - - for namespace in schema: - if len(namespace["fields"]) == 0: - continue - - for f in namespace["fields"]: - fields[f["name"]] = f - - return fields - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - parser.add_argument('--stdout', help='output to stdout instead of files') - args = parser.parse_args() - - # Outputs html of links to each use case (for the readme) - # and generates an html file per use case besides their each yaml file. - if args.stdout == "true": - write_stdout() diff --git a/use-cases/README.md b/use-cases/README.md deleted file mode 100644 index cb658c9432..0000000000 --- a/use-cases/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Use cases - -The use cases directory is used to define the fields for some more specific use -cases. All the fields used here are inherited from ECS but are referenced here -to have more details on it. - -## Generate - -Execute `make` in the project's root directory after modifying any of the `.yml` -files in this directory to rebuild the `.md` Markdown files. diff --git a/use-cases/apm.md b/use-cases/apm.md deleted file mode 100644 index 8a63ae0aa3..0000000000 --- a/use-cases/apm.md +++ /dev/null @@ -1,21 +0,0 @@ -## APM use case - -ECS usage for the APM data. - -### APM fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| *id* | *Unique id to describe the event.* | (use case) | keyword | `8a4f500d` | -| [@timestamp](../README.md#@timestamp) | Timestamp when the event was created in the app / service. | core | date | `2016-05-23T08:05:34.853Z` | -| *agent.** | *The agent fields are used to describe which agent did send the information.
* | | | | -| [agent.version](../README.md#agent.version) | APM Agent version. | core | keyword | `3.14.0` | -| [agent.name](../README.md#agent.name) | APM agent name. | core | keyword | `elastic-node` | -| *service.** | *The service fields describe the service inside which the APM agent is running.
* | | | | -| [service.id](../README.md#service.id) | Unique identifier of the running service. | core | keyword | `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` | -| [service.name](../README.md#service.name) | Name of the service the agent is running in. This is normally a user defined name. | core | keyword | `user-service` | -| [service.version](../README.md#service.version) | Version of the service the agent is running in. This depends on if the service is given a version. | core | keyword | `3.2.4` | - - - diff --git a/use-cases/apm.yml b/use-cases/apm.yml deleted file mode 100644 index 677ce3c996..0000000000 --- a/use-cases/apm.yml +++ /dev/null @@ -1,58 +0,0 @@ -title: APM -name: apm -description: - ECS usage for the APM data. -fields: -- name: base - fields: - - name: id - type: keyword - description: > - Unique id to describe the event. - example: 8a4f500d - - name: "@timestamp" - type: date - phase: 1 - example: "2016-05-23T08:05:34.853Z" - description: > - Timestamp when the event was created in the app / service. - -- name: agent - description: > - The agent fields are used to describe which agent did send the information. - fields: - - name: version - type: keyword - description: > - APM Agent version. - example: 3.14.0 - - name: name - type: keyword - description: > - APM agent name. - example: elastic-node - -- name: service - description: > - The service fields describe the service inside which the APM agent is running. - fields: - - name: id - type: keyword - description: > - Unique identifier of the running service. - example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 - - - name: name - type: keyword - example: "user-service" - description: > - Name of the service the agent is running in. This is normally a - user defined name. - - - name: version - type: keyword - example: "3.2.4" - description: > - Version of the service the agent is running in. This depends - on if the service is given a version. - diff --git a/use-cases/auditbeat.md b/use-cases/auditbeat.md deleted file mode 100644 index dff825a597..0000000000 --- a/use-cases/auditbeat.md +++ /dev/null @@ -1,44 +0,0 @@ -## Auditbeat use case - -ECS usage in Auditbeat. - -### Auditbeat fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| [event.module](../README.md#event.module) | Auditbeat module name. | core | keyword | `apache` | -| *file.** | *File attributes.
* | | | | -| [file.path](../README.md#file.path) | The path to the file. | extended | keyword | `/home/alice/example.png` | -| [file.target_path](../README.md#file.target_path) | The target path for symlinks. | extended | keyword | | -| [file.type](../README.md#file.type) | The file type (file, dir, or symlink). | extended | keyword | `file` | -| [file.device](../README.md#file.device) | The device. | extended | keyword | `sda` | -| [file.inode](../README.md#file.inode) | The inode representing the file in the filesystem. | extended | keyword | `256383` | -| [file.uid](../README.md#file.uid) | The user ID (UID) or security identifier (SID) of the file owner. | extended | keyword | `1001` | -| [file.owner](../README.md#file.owner) | The file owner's username. | extended | keyword | `alice` | -| [file.gid](../README.md#file.gid) | The primary group ID (GID) of the file. | extended | keyword | `1001` | -| [file.group](../README.md#file.group) | The primary group name of the file. | extended | keyword | `alice` | -| [file.mode](../README.md#file.mode) | The mode of the file in octal representation. | extended | keyword | `416` | -| [file.size](../README.md#file.size) | The file size in bytes (field is only added when `type` is `file`). | extended | long | `16384` | -| [file.mtime](../README.md#file.mtime) | The last modified time of the file (time when content was modified). | extended | date | | -| [file.ctime](../README.md#file.ctime) | The last change time of the file (time when metadata was changed). | extended | date | | -| *hash.** | *Hash fields used in Auditbeat.
The hash field contains cryptographic hashes of data associated with the event (such as a file). The keys are names of cryptographic algorithms. The values are encoded as hexidecimal (lower-case).
All fields in user can have one or multiple entries.
* | | | | -| *hash.blake2b_256* | *BLAKE2b-256 hash of the file.* | (use case) | keyword | | -| *hash.blake2b_384* | *BLAKE2b-384 hash of the file.* | (use case) | keyword | | -| *hash.blake2b_512* | *BLAKE2b-512 hash of the file.* | (use case) | keyword | | -| [hash.md5](../README.md#hash.md5) | MD5 hash. | extended | keyword | | -| [hash.sha1](../README.md#hash.sha1) | SHA-1 hash. | extended | keyword | | -| *hash.sha224* | *SHA-224 hash (SHA-2 family).* | (use case) | keyword | | -| [hash.sha256](../README.md#hash.sha256) | SHA-256 hash (SHA-2 family). | extended | keyword | | -| *hash.sha384* | *SHA-384 hash (SHA-2 family).* | (use case) | keyword | | -| [hash.sha512](../README.md#hash.sha512) | SHA-512 hash (SHA-2 family). | extended | keyword | | -| *hash.sha512_224* | *SHA-512/224 hash (SHA-2 family).* | (use case) | keyword | | -| *hash.sha512_256* | *SHA-512/256 hash (SHA-2 family).* | (use case) | keyword | | -| *hash.sha3_224* | *SHA3-224 hash (SHA-3 family).* | (use case) | keyword | | -| *hash.sha3_256* | *SHA3-256 hash (SHA-3 family).* | (use case) | keyword | | -| *hash.sha3_384* | *SHA3-384 hash (SHA-3 family).* | (use case) | keyword | | -| *hash.sha3_512* | *SHA3-512 hash (SHA-3 family).* | (use case) | keyword | | -| *hash.xxh64* | *XX64 hash of the file.* | (use case) | keyword | | - - - diff --git a/use-cases/auditbeat.yml b/use-cases/auditbeat.yml deleted file mode 100644 index b7b25fcb48..0000000000 --- a/use-cases/auditbeat.yml +++ /dev/null @@ -1,156 +0,0 @@ -title: Auditbeat -name: auditbeat -description: - ECS usage in Auditbeat. -fields: -- name: event - fields: - - name: module - description: > - Auditbeat module name. -- name: file - title: File - description: > - File attributes. - fields: - - name: path - type: keyword - description: The path to the file. - - - name: target_path - type: keyword - description: The target path for symlinks. - - - name: type - type: keyword - description: The file type (file, dir, or symlink). - - - name: device - type: keyword - description: The device. - - - name: inode - type: keyword - description: The inode representing the file in the filesystem. - - - name: uid - type: keyword - description: > - The user ID (UID) or security identifier (SID) of the file owner. - - - name: owner - type: keyword - description: The file owner's username. - - - name: gid - type: keyword - description: The primary group ID (GID) of the file. - - - name: group - type: keyword - description: The primary group name of the file. - - - name: mode - type: keyword - example: 0640 - description: The mode of the file in octal representation. - - - name: size - type: long - description: The file size in bytes (field is only added when `type` is `file`). - - - name: mtime - type: date - description: The last modified time of the file (time when content was modified). - - - name: ctime - type: date - description: The last change time of the file (time when metadata was changed). - - -- name: hash - group: 3 - description: > - Hash fields used in Auditbeat. - - The hash field contains cryptographic hashes of data associated with the event - (such as a file). The keys are names of cryptographic algorithms. The values - are encoded as hexidecimal (lower-case). - - All fields in user can have one or multiple entries. - fields: - - name: blake2b_256 - type: keyword - description: BLAKE2b-256 hash of the file. - - - name: blake2b_384 - type: keyword - description: BLAKE2b-384 hash of the file. - - - name: blake2b_512 - type: keyword - description: BLAKE2b-512 hash of the file. - - - name: md5 - type: keyword - description: > - MD5 hash. - - - name: sha1 - type: keyword - description: > - SHA-1 hash. - - - name: sha224 - type: keyword - description: > - SHA-224 hash (SHA-2 family). - - - name: sha256 - type: keyword - description: > - SHA-256 hash (SHA-2 family). - - - name: sha384 - type: keyword - description: > - SHA-384 hash (SHA-2 family). - - - name: sha512 - type: keyword - description: > - SHA-512 hash (SHA-2 family). - - - name: sha512_224 - type: keyword - description: > - SHA-512/224 hash (SHA-2 family). - - - name: sha512_256 - type: keyword - description: > - SHA-512/256 hash (SHA-2 family). - - - name: sha3_224 - type: keyword - description: > - SHA3-224 hash (SHA-3 family). - - - name: sha3_256 - type: keyword - description: > - SHA3-256 hash (SHA-3 family). - - - name: sha3_384 - type: keyword - description: > - SHA3-384 hash (SHA-3 family). - - - name: sha3_512 - type: keyword - description: > - SHA3-512 hash (SHA-3 family). - - - name: xxh64 - type: keyword - description: XX64 hash of the file. diff --git a/use-cases/beats.md b/use-cases/beats.md deleted file mode 100644 index c96e994b2d..0000000000 --- a/use-cases/beats.md +++ /dev/null @@ -1,18 +0,0 @@ -## Beats use case - -ECS fields used in Beats. - -### Beats fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| *id* | *Unique id to describe the event.* | (use case) | keyword | `8a4f500d` | -| *timestamp* | *Timestamp when the event was created.* | (use case) | date | `2016-05-23T08:05:34.853Z` | -| *agent.** | *The agent fields are used to describe by which beat the information was collected.
* | | | | -| [agent.version](../README.md#agent.version) | Beat version. | core | keyword | `6.0.0-rc2` | -| [agent.name](../README.md#agent.name) | Beat name. | core | keyword | `filebeat` | -| [agent.id](../README.md#agent.id) | Unique beat identifier. | core | keyword | `8a4f500d` | - - - diff --git a/use-cases/beats.yml b/use-cases/beats.yml deleted file mode 100644 index 92911bb4dd..0000000000 --- a/use-cases/beats.yml +++ /dev/null @@ -1,38 +0,0 @@ -title: Beats -name: beats -description: - ECS fields used in Beats. -fields: -- name: base - fields: - - name: id - type: keyword - description: > - Unique id to describe the event. - example: 8a4f500d - - name: timestamp - type: date - phase: 1 - example: "2016-05-23T08:05:34.853Z" - description: > - Timestamp when the event was created. - -- name: agent - description: > - The agent fields are used to describe by which beat the information was collected. - fields: - - name: version - type: keyword - description: > - Beat version. - example: 6.0.0-rc2 - - name: name - type: keyword - description: > - Beat name. - example: filebeat - - name: id - type: keyword - description: > - Unique beat identifier. - example: 8a4f500d diff --git a/use-cases/filebeat-apache-access.md b/use-cases/filebeat-apache-access.md deleted file mode 100644 index a9ef41840f..0000000000 --- a/use-cases/filebeat-apache-access.md +++ /dev/null @@ -1,29 +0,0 @@ -## Filebeat Apache use case - -ECS fields used in Filebeat for the apache module. - -### Filebeat Apache fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| *id* | *Unique id to describe the event.* | (use case) | keyword | `8a4f500d` | -| [@timestamp](../README.md#@timestamp) | Timestamp of the log line after processing. | core | date | `2016-05-23T08:05:34.853Z` | -| [message](../README.md#message) | Log message of the event | core | text | `Hello World` | -| [event.module](../README.md#event.module) | Currently fileset.module | core | keyword | `apache` | -| [event.dataset](../README.md#event.dataset) | Currenly fileset.name | core | keyword | `access` | -| [source.ip](../README.md#source.ip) | Source ip of the request. Currently apache.access.remote_ip | core | ip | `192.168.1.1` | -| [user.name](../README.md#user.name) | User name in the request. Currently apache.access.user_name | core | keyword | `ruflin` | -| *http.method* | *Http method, currently apache.access.method* | (use case) | keyword | `GET` | -| *http.url* | *Http url, currently apache.access.url* | (use case) | keyword | `http://elastic.co/` | -| [http.version](../README.md#http.version) | Http version, currently apache.access.http_version | extended | keyword | `1.1` | -| *http.response.code* | *Http response code, currently apache.access.response_code* | (use case) | keyword | `404` | -| *http.response.body_sent.bytes* | *Http response body bytes sent, currently apache.access.body_sent.bytes* | (use case) | long | `117` | -| *http.referer* | *Http referrer code, currently apache.access.referrer
NOTE: In the RFC its misspell as referer and has become accepted standard* | (use case) | keyword | `http://elastic.co/` | -| *user_agent.** | *User agent fields as in schema. Currently under apache.access.user_agent.*
* | | | | -| [user_agent.original](../README.md#user_agent.original) | Original user agent. Currently apache.access.agent | extended | keyword | `http://elastic.co/` | -| *geoip.** | *User agent fields as in schema. Currently under apache.access.geoip.*
These are extracted from source.ip
Should they be under source.geoip?
* | | | | -| *geoip....* | *All geoip fields.* | (use case) | keyword | | - - - diff --git a/use-cases/filebeat-apache-access.yml b/use-cases/filebeat-apache-access.yml deleted file mode 100644 index ac8c004095..0000000000 --- a/use-cases/filebeat-apache-access.yml +++ /dev/null @@ -1,111 +0,0 @@ -title: Filebeat Apache -name: filebeat-apache-access -description: - ECS fields used in Filebeat for the apache module. -fields: -- name: base - fields: - - name: id - type: keyword - description: > - Unique id to describe the event. - example: 8a4f500d - - name: "@timestamp" - type: date - example: "2016-05-23T08:05:34.853Z" - description: > - Timestamp of the log line after processing. - - name: message - type: date - example: "Hello World" - description: > - Log message of the event - -- name: event - fields: - - name: module - type: keyword - description: > - Currently fileset.module - example: apache - - name: dataset - type: keyword - example: access - description: > - Currenly fileset.name - -- name: source - fields: - - name: ip - type: ip - description: > - Source ip of the request. Currently apache.access.remote_ip - example: 192.168.1.1 - -- name: user - fields: - - name: name - type: keyword - description: > - User name in the request. Currently apache.access.user_name - example: ruflin - -- name: http - fields: - - name: method - type: keyword - description: > - Http method, currently apache.access.method - example: GET - - name: url - type: keyword - description: > - Http url, currently apache.access.url - example: "http://elastic.co/" - - name: version - type: keyword - description: > - Http version, currently apache.access.http_version - example: 1.1 - - name: response.code - type: keyword - description: > - Http response code, currently apache.access.response_code - example: 404 - - name: response.body_sent.bytes - type: long - description: > - Http response body bytes sent, currently apache.access.body_sent.bytes - example: 117 - - name: referer - type: keyword - description: > - Http referrer code, currently apache.access.referrer - - NOTE: In the RFC its misspell as referer and has become accepted standard - example: http://elastic.co/ - -- name: user_agent - title: User Agent - description: > - User agent fields as in schema. Currently under apache.access.user_agent.* - fields: - - name: original - type: keyword - description: > - Original user agent. Currently apache.access.agent - example: http://elastic.co/ - -- name: geoip - title: Geoip - description: > - User agent fields as in schema. Currently under apache.access.geoip.* - - These are extracted from source.ip - - Should they be under source.geoip? - fields: - - name: ... - type: keyword - description: > - All geoip fields. diff --git a/use-cases/kubernetes.md b/use-cases/kubernetes.md deleted file mode 100644 index 5588da6060..0000000000 --- a/use-cases/kubernetes.md +++ /dev/null @@ -1,21 +0,0 @@ -## Kubernetes use case - -You can monitor containers running in a Kubernetes cluster by adding Kubernetes-specific information under `kubernetes.` - - -### Kubernetes fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| [container.id](../README.md#container.id) | Unique container id. | core | keyword | `fdbef803fa2b` | -| [container.name](../README.md#container.name) | Container name. | extended | keyword | | -| [host.hostname](../README.md#host.hostname) | Hostname of the host.
It normally contains what the `hostname` command returns on the host machine. | core | keyword | `kube-high-cpu-42` | -| *kubernetes.pod.name* | *Kubernetes pod name* | (use case) | keyword | `foo-webserver` | -| *kubernetes.namespace* | *Kubernetes namespace* | (use case) | keyword | `foo-team` | -| *kubernetes.labels* | *Kubernetes labels map* | (use case) | object | | -| *kubernetes.annotations* | *Kubernetes annotations map* | (use case) | object | | -| *kubernetes.container.name* | *Kubernetes container name. This name is unique within the pod only. It is different from the `container.name` field.* | (use case) | keyword | | - - - diff --git a/use-cases/kubernetes.yml b/use-cases/kubernetes.yml deleted file mode 100644 index f4c46317f8..0000000000 --- a/use-cases/kubernetes.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: kubernetes -title: Kubernetes -description: > - You can monitor containers running in a Kubernetes cluster by adding - Kubernetes-specific information under `kubernetes.` - -fields: -- name: container - fields: - - - name: id - example: fdbef803fa2b - - - name: name - -- name: host - fields: - - - name: hostname - example: kube-high-cpu-42 - -- name: kubernetes - fields: - - - name: pod.name - type: keyword - description: > - Kubernetes pod name - example: foo-webserver - - - name: namespace - type: keyword - description: > - Kubernetes namespace - example: foo-team - - - name: labels - type: object - description: > - Kubernetes labels map - - - name: annotations - type: object - description: > - Kubernetes annotations map - - - name: container.name - type: keyword - description: > - Kubernetes container name. This name is unique within the pod only. - It is different from the `container.name` field. diff --git a/use-cases/logging.md b/use-cases/logging.md deleted file mode 100644 index be7efd0b6b..0000000000 --- a/use-cases/logging.md +++ /dev/null @@ -1,22 +0,0 @@ -## Logging use case - -ECS fields used in logging use cases. - -### Logging fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| *id* | *Unique id of the log entry.* | (use case) | keyword | `8a4f500d` | -| *timestamp* | *Timestamp of the log line.* | (use case) | date | `2016-05-23T08:05:34.853Z` | -| [message](../README.md#message) | The log message.
This can contain the full log line or based on the processing only the extracted message part. This is expected to be human readable. | core | text | `Hello World` | -| *hostname* | *Hostname extracted from the log line.* | (use case) | keyword | `www.example.com` | -| *ip* | *IP Address extracted from the log line. Can be IPv4 or IPv6.* | (use case) | ip | `192.168.1.12` | -| [log.level](../README.md#log.level) | Log level field. Is expected to be `WARN`, `ERR`, `INFO` etc. | core | keyword | `ERR` | -| *log.line* | *Line number the log event was collected from.* | (use case) | long | `18` | -| *log.offset* | *Offset of the log event.* | (use case) | long | `12` | -| *source.** | *Describes from where the log entries come from.
* | | | | -| *source.path* | *File path of the file the data is harvested from.* | (use case) | keyword | `/var/log/test.log` | - - - diff --git a/use-cases/logging.yml b/use-cases/logging.yml deleted file mode 100644 index f307ced148..0000000000 --- a/use-cases/logging.yml +++ /dev/null @@ -1,67 +0,0 @@ -title: Logging -name: logging -description: - ECS fields used in logging use cases. -fields: -- name: base - fields: - - name: id - type: keyword - description: > - Unique id of the log entry. - example: 8a4f500d - - name: timestamp - type: date - example: "2016-05-23T08:05:34.853Z" - description: > - Timestamp of the log line. - - name: message - type: text - required: true - example: "Hello World" - description: > - The log message. - - This can contain the full log line or based on the processing - only the extracted message part. This is expected to be human readable. - - - name: hostname - type: keyword - example: "www.example.com" - description: > - Hostname extracted from the log line. - - name: ip - type: ip - example: "192.168.1.12" - description: > - IP Address extracted from the log line. Can be IPv4 or IPv6. - - -- name: log - fields: - - name: level - type: keyword - description: > - Log level field. Is expected to be `WARN`, `ERR`, `INFO` etc. - example: ERR - - name: line - type: long - description: > - Line number the log event was collected from. - example: 18 - - name: offset - type: long - description: > - Offset of the log event. - example: 12 - - -- name: source - description: > - Describes from where the log entries come from. - fields: - - name: path - type: keyword - description: > - File path of the file the data is harvested from. - example: /var/log/test.log diff --git a/use-cases/metricbeat.md b/use-cases/metricbeat.md deleted file mode 100644 index c573a7897e..0000000000 --- a/use-cases/metricbeat.md +++ /dev/null @@ -1,31 +0,0 @@ -## Metricbeat use case - -ECS fields used Metricbeat. - -### Metricbeat fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| *id* | *Unique id to describe the event.* | (use case) | keyword | `8a4f500d` | -| *timestamp* | *Timestamp when the event was created.* | (use case) | date | `2016-05-23T08:05:34.853Z` | -| [agent.version](../README.md#agent.version) | Beat version. | core | keyword | `6.0.0-rc2` | -| [agent.name](../README.md#agent.name) | Beat name. | core | keyword | `filebeat` | -| [agent.id](../README.md#agent.id) | Unique beat identifier. | core | keyword | `8a4f500d` | -| *service.** | *The service fields describe the service for / from which the data was collected.
If logs or metrics are collected from Redis, `service.name` would be `redis`. This allows to find and correlate logs for a specicic service or even version with `service.version`.
* | | | | -| [service.id](../README.md#service.id) | Unique identifier of the running service.
This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. For example in case of issues with one redis instance, it's possible to filter on the id to see metrics and logs for this single instance. | core | keyword | `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` | -| [service.name](../README.md#service.name) | Name of the service data is collected from.
The name is normally the same as the module name. | core | keyword | `elasticsearch` | -| [service.version](../README.md#service.version) | Version of the service the data was collected from.
This allows to look at a data set only for a specific version of a service. | core | keyword | `3.2.4` | -| *service.host* | *Host address that is used to connect to the service.
This normally contains hostname + port.
REVIEW: Should this be service.uri instead, sometimes it's more then just the host? It could also include a path or the protocol.* | (use case) | keyword | `elasticsearch:9200` | -| *request.rtt* | *Request round trip time.
How long did the request take to fetch metrics from the service.
REVIEW: THIS DOES NOT EXIST YET IN ECS.* | (use case) | long | `115` | -| *error.** | *Error namespace
Use for errors which can happen during fetching information for a service.
* | | | | -| [error.message](../README.md#error.message) | Error message returned by the service during fetching metrics. | core | text | | -| [error.code](../README.md#error.code) | Error code returned by the service during fetching metrics. | core | keyword | | -| [host.hostname](../README.md#host.hostname) | Hostname of the system metricbeat is running on or user defined name. | core | keyword | | -| *host.timezone.offset.sec* | *Timezone offset of the host in seconds.* | (use case) | long | | -| [host.id](../README.md#host.id) | Unique host id. | core | keyword | | -| [event.module](../README.md#event.module) | Name of the module this data is coming from. | core | keyword | `mysql` | -| [event.dataset](../README.md#event.dataset) | Name of the dataset.
This contains the information which is currently stored in metricset.name and metricset.module. | core | keyword | `stats` | - - - diff --git a/use-cases/metricbeat.yml b/use-cases/metricbeat.yml deleted file mode 100644 index 74b8217fd4..0000000000 --- a/use-cases/metricbeat.yml +++ /dev/null @@ -1,146 +0,0 @@ -title: Metricbeat -name: metricbeat -description: - ECS fields used Metricbeat. -fields: -- name: base - fields: - - name: id - type: keyword - description: > - Unique id to describe the event. - example: 8a4f500d - - name: timestamp - type: date - phase: 1 - example: "2016-05-23T08:05:34.853Z" - description: > - Timestamp when the event was created. - -- name: agent - fields: - - name: version - type: keyword - description: > - Beat version. - example: 6.0.0-rc2 - - name: name - type: keyword - description: > - Beat name. - example: filebeat - - name: id - type: keyword - description: > - Unique beat identifier. - example: 8a4f500d - -- name: service - description: > - The service fields describe the service for / from which the data was collected. - - If logs or metrics are collected from Redis, `service.name` would be `redis`. This allows - to find and correlate logs for a specicic service or even version with `service.version`. - - fields: - - name: id - type: keyword - description: > - Unique identifier of the running service. - - This id should uniquely identify this service. This makes it possible - to correlate logs and metrics for one specific service. For example - in case of issues with one redis instance, it's possible to filter on the id - to see metrics and logs for this single instance. - - example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 - - - name: name - type: keyword - example: "elasticsearch" - description: > - Name of the service data is collected from. - - The name is normally the same as the module name. - - - name: version - type: keyword - example: "3.2.4" - description: > - Version of the service the data was collected from. - - This allows to look at a data set only for a specific version of a service. - - - name: host - type: keyword - example: "elasticsearch:9200" - description: > - Host address that is used to connect to the service. - - This normally contains hostname + port. - - REVIEW: Should this be service.uri instead, sometimes it's more then just the host? - It could also include a path or the protocol. - -- name: request - fields: - - name: rtt - type: long - description: > - Request round trip time. - - How long did the request take to fetch metrics from the service. - - REVIEW: THIS DOES NOT EXIST YET IN ECS. - - example: 115 - -- name: error - description: > - Error namespace - - Use for errors which can happen during fetching information for a service. - fields: - - name: message - type: text - description: > - Error message returned by the service during fetching metrics. - - - name: code - type: long - description: > - Error code returned by the service during fetching metrics. - -- name: host - fields: - - name: hostname - type: keyword - description: > - Hostname of the system metricbeat is running on or user defined name. - - - name: timezone.offset.sec - type: long - description: > - Timezone offset of the host in seconds. - - - name: id - type: keyword - description: > - Unique host id. - -- name: event - fields: - - name: module - type: keyword - description: > - Name of the module this data is coming from. - example: mysql - - name: dataset - type: keyword - description: > - Name of the dataset. - - This contains the information which is currently stored in metricset.name - and metricset.module. - - example: stats diff --git a/use-cases/web-logs.md b/use-cases/web-logs.md deleted file mode 100644 index 57f9a96062..0000000000 --- a/use-cases/web-logs.md +++ /dev/null @@ -1,29 +0,0 @@ -## Parsing web server logs use case - -Representing web server access logs in ECS. -This use case uses previous definitions for `http` and `user_agent` fields sets, which were taken out of ECS temporarily for Beta1. Their official definition in ECS is expected to change slightly. -Using the fields as represented here is not expected to conflict with ECS, but may require a transition, when they are re-introduced officially. - -### Parsing web server logs fields - - -| Field | Description | Level | Type | Example | -|---|---|---|---|---| -| [@timestamp](../README.md#@timestamp) | Time at which the response was sent, and the web server log created. | core | date | `2016-05-23T08:05:34.853Z` | -| *http.** | *Fields related to HTTP requests and responses.
* | | | | -| [http.request.method](../README.md#http.request.method) | Http request method. | extended | keyword | `GET, POST, PUT` | -| [http.request.referrer](../README.md#http.request.referrer) | Referrer for this HTTP request. | extended | keyword | `https://blog.example.com/` | -| [http.response.status_code](../README.md#http.response.status_code) | Http response status code. | extended | long | `404` | -| [http.response.body.content](../README.md#http.response.body.content) | The full http response body. | extended | keyword | `Hello world` | -| [http.version](../README.md#http.version) | Http version. | extended | keyword | `1.1` | -| *user_agent.** | *The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string.
* | | | | -| [user_agent.original](../README.md#user_agent.original) | Unparsed version of the user_agent. | extended | keyword | `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` | -| *user_agent.device* | *Name of the physical device.* | (use case) | keyword | | -| [user_agent.version](../README.md#user_agent.version) | Version of the physical device. | extended | keyword | `12.0` | -| *user_agent.major* | *Major version of the user agent.* | (use case) | long | | -| *user_agent.minor* | *Minor version of the user agent.* | (use case) | long | | -| *user_agent.patch* | *Patch version of the user agent.* | (use case) | keyword | | -| [user_agent.name](../README.md#user_agent.name) | Name of the user agent. | extended | keyword | `Chrome` | - - - diff --git a/use-cases/web-logs.yml b/use-cases/web-logs.yml deleted file mode 100644 index 06a8f47d9e..0000000000 --- a/use-cases/web-logs.yml +++ /dev/null @@ -1,111 +0,0 @@ -title: Parsing web server logs -name: web-logs -description: - Representing web server access logs in ECS. - - This use case uses previous definitions for `http` and `user_agent` fields sets, - which were taken out of ECS temporarily for Beta1. Their official definition - in ECS is expected to change slightly. - - Using the fields as represented here is not expected to conflict with ECS, - but may require a transition, when they are re-introduced officially. -fields: - -- name: base - fields: - - name: "@timestamp" - type: date - phase: 1 - example: "2016-05-23T08:05:34.853Z" - description: > - Time at which the response was sent, and the web server log created. - -- name: http - title: HTTP - group: 2 - description: > - Fields related to HTTP requests and responses. - type: group - fields: - - - name: request.method - type: keyword - description: > - Http request method. - example: GET, POST, PUT - - - name: request.referrer - type: keyword - description: > - Referrer for this HTTP request. - example: https://blog.example.com/ - - - name: response.status_code - type: long - description: > - Http response status code. - example: 404 - - - name: response.body.content - type: keyword - description: > - The full http response body. - example: Hello world - - - name: version - type: keyword - description: > - Http version. - example: 1.1 - -- name: user_agent - title: User agent - group: 2 - description: > - The user_agent fields normally come from a browser request. They often - show up in web service logs coming from the parsed user agent string. - type: group - fields: - - - name: original - level: extended - type: keyword - description: > - Unparsed version of the user_agent. - - - name: device - level: extended - type: keyword - description: > - Name of the physical device. - - - name: version - level: extended - type: keyword - description: > - Version of the physical device. - - - name: major - level: extended - type: long - description: > - Major version of the user agent. - - - name: minor - level: extended - type: long - description: > - Minor version of the user agent. - - - name: patch - level: extended - type: keyword - description: > - Patch version of the user agent. - - - name: name - level: extended - type: keyword - example: Chrome - description: > - Name of the user agent.