From d66128ff383c34e98f30fbbf058e954dfe386f34 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 3 Jun 2020 18:27:55 -0400 Subject: [PATCH] Add details/examples to the httpjson docs (#18920) This adds some configuration examples to the documentation for the Filebeat httpjson input and adds some more detail to the configuration options. --- .../docs/inputs/input-httpjson.asciidoc | 174 +++++++++++++----- 1 file changed, 131 insertions(+), 43 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 441bcde7f6e..61f1d249198 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -13,39 +13,39 @@ beta[] Use the `httpjson` input to read messages from an HTTP API with JSON payloads. -For example, this input is used to retrieve MISP threat indicators in the filebeat -x-pack misp module. +For example, this input is used to retrieve MISP threat indicators in the +Filebeat <> module. -This input supports timed retrieval at a configurable interval and pagination. +This input supports retrieval at a configurable interval and pagination. Example configurations: ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: +# Fetch your public IP every minute. - type: httpjson - api_key: - interval: 12h - url: + url: https://api.ipify.org/?format=json + interval: 1m + processors: + - decode_json_fields + fields: [message] + target: json ---- ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: - type: httpjson + url: http://localhost:9200/_search?scroll=5m http_method: POST json_objects_array: hits.hits - pagination: |- - { - "enabled": true, - "extra_body_content": { - "scroll": "5m" - }, - "id_field": "_scroll_id", - "req_field": "scroll_id", - "url": "http://localhost:9200/_search/scroll" - } - url: http://localhost:9200/filebeat-test/_search?scroll=5m + pagination: + extra_body_content: + scroll: 5m + id_field: _scroll_id + req_field: scroll_id + url: http://localhost:9200/_search/scroll ---- @@ -57,104 +57,192 @@ The `httpjson` input supports the following configuration options plus the [float] ==== `api_key` -API key to access the HTTP API. Optional. Default: "", which means no API key is used. +API key to access the HTTP API. When set, this adds an `Authorization` header to +the HTTP request with this as the value. [float] ==== `http_client_timeout` -Time duration before declaring that the HTTP client connection has timed out. Default: 60s. +Duration before declaring that the HTTP client connection has timed out. +Defaults to `60s`. Valid time units are `ns`, `us`, `ms`, `s` (default), `m`, +`h`. [float] ==== `http_headers` -Extra HTTP header options can be set by specifying this JSON object. Default: not used. +Additional HTTP headers to set in the requests. The default value is `null` +(no additional headers). + +["source","yaml",subs="attributes"] +---- +- type: httpjson + http_headers: + Authorization: 'Basic aGVsbG86d29ybGQ=' +---- [float] ==== `http_method` -This option specifies which HTTP method to use, GET or POST. Default: GET. +HTTP method to use when making requests. `GET` or `POST` are the options. +Defaults to `GET`. [float] ==== `http_request_body` -Any additional data that needs to be set in the HTTP request can be specified in -this JSON blob. Default: not used. +An optional HTTP POST body. The configuration value must be an object, and it +will be encoded to JSON. This is only valid when `http_method` is `POST`. +Defaults to `null` (no HTTP body). + +["source","yaml",subs="attributes"] +---- +- type: httpjson + http_method: POST + http_request_body: + query: + bool: + filter: + term: + type: authentication +---- [float] ==== `interval` -Time duration between repeated data retrievals. Default: 0s, meaning no repeated data retrieval. +Duration between repeated requests. By default, the interval is `0` which means +it performs a single request then stops. It may make additional pagination +requests in response to the initial request if pagination is enabled. [float] ==== `json_objects_array` -If the HTTP API returns data in a JSON array, then this option can be set to decode these records -from the array. Default: not used. +If the response body contains a JSON object containing an array then this option +specifies the key containing that array. Each object in that array will generate +an event. This example response contains an array called `events` that we want +to index. + +["source","json",subs="attributes"] +---- +{ + "time": "2020-06-02 23:22:32 UTC", + "events": [ + { + "timestamp": "2020-05-02 11:10:03 UTC", + "event": { + "category": "authorization" + }, + "user": { + "name": "fflintstone" + } + }, + { + "timestamp": "2020-05-05 13:03:11 UTC", + "event": { + "category": "authorization" + }, + "user": { + "name": "brubble" + } + } + ] +} +---- + +The config needs to specify `events` as the `json_objects_array` value. + +["source","yaml",subs="attributes"] +---- +- type: httpjson + json_objects_array: events +---- [float] ==== `no_http_body` -If set, do not use HTTP request body. Default: false. +Force HTTP requests to be sent with an empty HTTP body. Defaults to `false`. +This option cannot be used with `http_request_body`, +`pagination.extra_body_content`, or `pagination.req_field`. [float] ==== `pagination.enabled` -This option specifies whether pagination is enabled. Default: true. +The `enabled` setting can be used to disable the pagination configuration by +setting it to `false`. The default value is `true`. + +NOTE: Pagination settings are disabled if either `enabled` is set to `false` or +the `pagination` section is missing. [float] ==== `pagination.extra_body_content` -Any additional data that needs to be set in the HTTP pagination request can be specified in -this JSON blob. Default: not used. +An object containing additional fields that should be included in the pagination +request body. Defaults to `null`. + +["source","yaml",subs="attributes"] +---- +- type: httpjson + pagination.extra_body_content: + max_items: 500 +---- [float] ==== `pagination.header.field_name` -The field name in the HTTP Header that is used for pagination control. +The name of the HTTP header in the response that is used for pagination control. +The header value will be extracted from the response and used to make the next +pagination response. `pagination.header.regex_pattern` can be used to select +a subset of the value. [float] ==== `pagination.header.regex_pattern` -The regular expression pattern to use for retrieving the pagination information from the HTTP Header field specified above. +The regular expression pattern to use for retrieving the pagination information +from the HTTP header field specified above. The first match becomes as the +value. [float] ==== `pagination.id_field` -This specifies which field to use as the pagination id, to retrieve the id from the pagination -result JSON document. Required when pagination is enabled. +The name of a field in the JSON response body to use as the pagination ID. +The value will be included in the next pagination request under the key +specified by the `pagination.req_field` value. [float] ==== `pagination.req_field` -This specifies which field to use as the pagination request id, to send to the HTTP API. -Required when pagination is enabled. +The name of the field to include in the pagination JSON request body containing +the pagination ID defined by the `pagination.id_field` field. [float] ==== `pagination.url` -This specifies the URL for sending pagination request. Required if the pagination URL is different -than the HTTP API URL. +This specifies the URL for sending pagination requests. Defaults to the `url` +value. This is only needed when the pagination requests need to be routed to +a different URL. [float] ==== `rate_limit.limit` -This specifies the field in the HTTP Header of the response that specifies the total limit. +This specifies the field in the HTTP header of the response that specifies the +total limit. [float] ==== `rate_limit.remaining` -This specifies the field in the HTTP Header of the response that specifies the remaining quota of the rate limit. +This specifies the field in the HTTP header of the response that specifies the +remaining quota of the rate limit. [float] ==== `rate_limit.reset` -This specifies the field in the HTTP Header of the response that specifies the epoch time -when the rate limit will be reset. +This specifies the field in the HTTP Header of the response that specifies the +epoch time when the rate limit will reset. [float] ==== `ssl` -This specifies SSL/TLS common config. Default: not used. +This specifies SSL/TLS configuration. If the ssl section is missing, the host's +CAs are used for HTTPS connections. See <> for more +information. [float] ==== `url`