From f8f25adc8af231d66131a6961832080712e93a0f Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Fri, 19 May 2023 10:06:31 +0930 Subject: [PATCH] x-pack/filebeat/input/httpjson: provide XML decoding functionality (#35235) This addition makes use of the mito XML library to allow users to decode XML responses with type hints. --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 59 ++++++-- go.mod | 15 +- go.sum | 32 ++-- .../filebeat/docs/inputs/input-cel.asciidoc | 142 +++++++++--------- .../docs/inputs/input-httpjson.asciidoc | 9 +- .../input/httpjson/config_response.go | 1 + x-pack/filebeat/input/httpjson/encoding.go | 17 +++ x-pack/filebeat/input/httpjson/input.go | 12 +- x-pack/filebeat/input/httpjson/input_test.go | 99 ++++++++++++ x-pack/filebeat/input/httpjson/pagination.go | 8 +- x-pack/filebeat/input/httpjson/request.go | 14 +- .../filebeat/input/httpjson/request_test.go | 2 +- x-pack/filebeat/input/httpjson/response.go | 21 ++- 14 files changed, 313 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 308158d685d..aab047a1a26 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -283,6 +283,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Add oracle authentication messages parsing {pull}35127[35127] - Add support for CRC validation in Filebeat's HTTP endpoint input. {pull}35204[35204] - Add execution budget to CEL input. {pull}35409[35409] +- Add XML decoding support to HTTPJSON. {issue}34438[34438] {pull}35235[35235] *Auditbeat* - Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817] diff --git a/NOTICE.txt b/NOTICE.txt index 3f9042d71cc..e800db270ea 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13412,11 +13412,11 @@ limitations under the License. -------------------------------------------------------------------------------- Dependency : github.com/elastic/mito -Version: v0.0.0-20230302005114-1dda06e81678 +Version: v1.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.1.0/LICENSE: Apache License @@ -22588,11 +22588,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.7.0 +Version: v0.9.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.7.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.9.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -22625,11 +22625,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/oauth2 -Version: v0.0.0-20221014153046-6fdb5e3db783 +Version: v0.7.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/oauth2@v0.0.0-20221014153046-6fdb5e3db783/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/oauth2@v0.7.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -22699,11 +22699,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.6.0 +Version: v0.7.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.7.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -22736,11 +22736,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.7.0 +Version: v0.9.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.7.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.9.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -22810,11 +22810,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/tools -Version: v0.1.12 +Version: v0.6.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.1.12/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.6.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25184,6 +25184,37 @@ Public License instead of this License. Indirect dependencies +-------------------------------------------------------------------------------- +Dependency : aqwari.net/xml +Version: v0.0.0-20210331023308-d9421b293817 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/aqwari.net/xml@v0.0.0-20210331023308-d9421b293817/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 David Arroyo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/compute Version: v1.12.1 @@ -46725,11 +46756,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.5.0 +Version: v0.7.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.7.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index 24aad62c953..a7cbdd7b58a 100644 --- a/go.mod +++ b/go.mod @@ -157,13 +157,13 @@ require ( golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.9.0 - golang.org/x/net v0.7.0 - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.6.0 - golang.org/x/text v0.7.0 + golang.org/x/sys v0.7.0 + golang.org/x/text v0.9.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.1.12 + golang.org/x/tools v0.6.0 google.golang.org/api v0.102.0 google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c google.golang.org/grpc v1.50.1 @@ -199,7 +199,7 @@ require ( github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.6.0 github.com/elastic/go-elasticsearch/v8 v8.2.0 - github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678 + github.com/elastic/mito v1.1.0 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 github.com/google/cel-go v0.13.0 @@ -216,6 +216,7 @@ require ( ) require ( + aqwari.net/xml v0.0.0-20210331023308-d9421b293817 // indirect cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.6.0 // indirect @@ -335,7 +336,7 @@ require ( go.elastic.co/fastjson v1.1.0 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.5.0 // indirect + golang.org/x/term v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 29848fbef3a..f0d688f82d3 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +aqwari.net/xml v0.0.0-20210331023308-d9421b293817 h1:+3Rh5EaTzNLnzWx3/uy/mAaH/dGI7svJ6e0oOIDcPuE= +aqwari.net/xml v0.0.0-20210331023308-d9421b293817/go.mod h1:c7kkWzc7HS/t8Q2DcVY8P2d1dyWNEhEVT5pL0ZHO11c= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -668,8 +670,8 @@ github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11 github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678 h1:+meybiVLjeeEOsxW7Wuq05uOZQ874sVBMgxoGzUvg7E= -github.com/elastic/mito v0.0.0-20230302005114-1dda06e81678/go.mod h1:7dZE6T9eC4uwo6HG4OHbHDy255spws7MCwA2KBkppkU= +github.com/elastic/mito v1.1.0 h1:bgd2C3Vnnfd2CgXU0UtzVzUWonTMoRYM8zPkEFmvFj8= +github.com/elastic/mito v1.1.0/go.mod h1:tk+3dy/mzYVZfgYRqRFVxLfwrIL2kRpd3WeB+IB1ySI= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw= github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM= @@ -2044,6 +2046,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -2070,8 +2073,8 @@ golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2090,8 +2093,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2242,15 +2245,15 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2261,8 +2264,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2344,6 +2347,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200821192610-3366bbee4705/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2359,8 +2363,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index 43ce0f9e583..3c2bfede4d3 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -122,7 +122,7 @@ The field should be an array, but in the case of an error condition in the CEL p <2> If `cursor` is present it must be either be a single object or an array with the same length as events; each element _i_ of the `cursor` will be the details for obtaining the events at and beyond event _i_ in the `events` array. If the `cursor` is a single object it is will be the details for obtaining events after the last event in the `events` array and will only be retained on successful publication of all the events in the `events` array. -<3> If `rate_limit` is present it must be a map with numeric fields `rate` and `burst`. The `rate_limit` field may also have a string `error` field and other fields which will be logged. If it has an `error` field, the `rate` and `burst` will not be used to set rate limit behavior. The https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Limit[Limit], and https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#OktaRateLimit[Okta Rate Limit policy] and https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#DraftRateLimit[Draft Rate Limit policy] documentation show how to construct this field. +<3> If `rate_limit` is present it must be a map with numeric fields `rate` and `burst`. The `rate_limit` field may also have a string `error` field and other fields which will be logged. If it has an `error` field, the `rate` and `burst` will not be used to set rate limit behavior. The https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Limit[Limit], and https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#OktaRateLimit[Okta Rate Limit policy] and https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#DraftRateLimit[Draft Rate Limit policy] documentation show how to construct this field. <4> The evaluation is repeated with the new state, after removing the events field, if the "want_more" field is present and true, and a non-zero events array is returned. @@ -138,76 +138,76 @@ This will include any sensitive or secret information kept in the `state` object As noted above the `cel` input provides function, macro and global variables to extend the language. -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Collections[Collections] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Collate[Collate] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Drop[Drop] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Drop_Empty[Drop Empty] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Flatten[Flatten] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Max[Max] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Min[Min] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-With[With] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-With_Replace[With Replace] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-With_Update[With Update] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Crypto[Crypto] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Base64[Base64] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Base64_Raw[Base64 Raw] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Hex[Hex] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-SHA_1[SHA-1] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-SHA_256[SHA-256] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-HMAC[HMAC] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-UUID[UUID] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#File[File] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Dir[Dir] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-File[File] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#HTTP[HTTP] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-HEAD[HEAD] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-GET[GET] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-GET_Request[GET Request] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-POST[POST] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-POST_Request[POST Request] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Request[Request] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Basic_Authentication[Basic Authentication] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Do_Request[Do Request] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Parse_URL[Parse URL] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Format_URL[Format URL] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Parse_Query[Parse Query] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Format_Query[Format Query] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#File[File] — the file extension is initialized with MIME handlers for "application/gzip", https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#NDJSON["application/x-ndjson"], https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Zip["application/zip"], https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#CSVNoHeader["text/csv; header=absent"], and https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#CSVHeader["text/csv; header=present"]. -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Encode_JSON[Encode JSON] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Decode_JSON[Decode JSON] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Decode_JSON_Stream[Decode JSON Stream] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Limit[Limit] — the rate limit extension is initialized with https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#OktaRateLimit[Okta (as "okta")] and the https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#DraftRateLimit[Draft Rate Limit (as "draft")] policies. -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Rate_Limit[Rate Limit] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#MIME[MIME] — the MIME extension is initialized with MIME handlers for "application/gzip", https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#NDJSON["application/x-ndjson"], https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Zip["application/zip"], https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#CSVNoHeader["text/csv; header=absent"], and https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#CSVHeader["text/csv; header=present"]. -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-MIME[MIME] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Regexp[Regexp] — the regular expression extension is initialized with the patterns specified in the user input configuration via the `regexp` field. -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Match[RE Match] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Find[RE Find] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Find_All[RE Find All] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Find_Submatch[RE Find Submatch] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Find_All_Submatch[RE Find All Submatch] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-RE_Replace_All[RE Replace All] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Strings[Strings] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-String_Methods[String Methods] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-String_List_Methods[String List Methods] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Bytes_Methods[Bytes Methods] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Time[Time] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Format[Format] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Parse_Time[Parse Time] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Global_Variables[Global Variables] - -* https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#Try[Try] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Try[Try] -** https://pkg.go.dev/github.com/elastic/mito@v0.0.0-20230302005114-1dda06e81678/lib#hdr-Is_Error[Is Error] +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Collections[Collections] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Collate[Collate] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Drop[Drop] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Drop_Empty[Drop Empty] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Flatten[Flatten] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Max[Max] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Min[Min] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-With[With] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-With_Replace[With Replace] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-With_Update[With Update] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Crypto[Crypto] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Base64[Base64] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Base64_Raw[Base64 Raw] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Hex[Hex] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-SHA_1[SHA-1] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-SHA_256[SHA-256] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-HMAC[HMAC] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-UUID[UUID] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#File[File] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Dir[Dir] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-File[File] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#HTTP[HTTP] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-HEAD[HEAD] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-GET[GET] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-GET_Request[GET Request] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-POST[POST] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-POST_Request[POST Request] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Request[Request] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Basic_Authentication[Basic Authentication] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Do_Request[Do Request] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Parse_URL[Parse URL] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Format_URL[Format URL] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Parse_Query[Parse Query] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Format_Query[Format Query] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#File[File] — the file extension is initialized with MIME handlers for "application/gzip", https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#NDJSON["application/x-ndjson"], https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Zip["application/zip"], https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#CSVNoHeader["text/csv; header=absent"], and https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#CSVHeader["text/csv; header=present"]. +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Encode_JSON[Encode JSON] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Decode_JSON[Decode JSON] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Decode_JSON_Stream[Decode JSON Stream] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Limit[Limit] — the rate limit extension is initialized with https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#OktaRateLimit[Okta (as "okta")] and the https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#DraftRateLimit[Draft Rate Limit (as "draft")] policies. +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Rate_Limit[Rate Limit] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#MIME[MIME] — the MIME extension is initialized with MIME handlers for "application/gzip", https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#NDJSON["application/x-ndjson"], https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Zip["application/zip"], https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#CSVNoHeader["text/csv; header=absent"], and https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#CSVHeader["text/csv; header=present"]. +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-MIME[MIME] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Regexp[Regexp] — the regular expression extension is initialized with the patterns specified in the user input configuration via the `regexp` field. +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Match[RE Match] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Find[RE Find] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Find_All[RE Find All] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Find_Submatch[RE Find Submatch] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Find_All_Submatch[RE Find All Submatch] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-RE_Replace_All[RE Replace All] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Strings[Strings] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-String_Methods[String Methods] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-String_List_Methods[String List Methods] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Bytes_Methods[Bytes Methods] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Time[Time] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Format[Format] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Parse_Time[Parse Time] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Global_Variables[Global Variables] + +* https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#Try[Try] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Try[Try] +** https://pkg.go.dev/github.com/elastic/mito@v1.1.0/lib#hdr-Is_Error[Is Error] In addition to the extensions provided in the packages listed above, a global variable `useragent` is also provided which gives the user CEL program access to the {beatname_lc} user-agent string. diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 87efc658dc0..dc90ffdf525 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -655,12 +655,19 @@ This determines whether rotated logs should be gzip compressed. [float] ==== `response.decode_as` -ContentType used for decoding the response body. If set it will force the decoding in the specified format regardless of the `Content-Type` header value, otherwise it will honor it if possible or fallback to `application/json`. Supported values: `application/json, application/x-ndjson`, `text/csv`, `application/zip`. It is not set by default. +ContentType used for decoding the response body. If set it will force the decoding in the specified format regardless of the `Content-Type` header value, otherwise it will honor it if possible or fallback to `application/json`. Supported values: `application/json, application/x-ndjson`, `text/csv`, `application/zip`, `application/xml` and `text/xml`. It is not set by default. NOTE: For `text/csv`, one event for each line will be created, using the header values as the object keys. For this reason is always assumed that a header exists. NOTE: For `application/zip`, the `zip` file is expected to contain one or more `.json` or `.ndjson` files. The contents of all of them will be merged into a single list of `JSON` objects. +NOTE: For `application/xml` and `text/xml` type information for decoding the XML document can be provided via the `response.xsd` option. + +[float] +==== `response.xsd` + +XML documents may require additional type information to enable correct parsing and ingestion. This information can be provided as an XML Schema Definition (XSD) for the document using the `response.xsd` option. + [[response-transforms]] [float] ==== `response.transforms` diff --git a/x-pack/filebeat/input/httpjson/config_response.go b/x-pack/filebeat/input/httpjson/config_response.go index 61900c7de5a..2f0da5685f3 100644 --- a/x-pack/filebeat/input/httpjson/config_response.go +++ b/x-pack/filebeat/input/httpjson/config_response.go @@ -17,6 +17,7 @@ const ( type responseConfig struct { DecodeAs string `config:"decode_as"` + XSD string `config:"xsd"` RequestBodyOnPagination bool `config:"request_body_on_pagination"` Transforms transformsConfig `config:"transforms"` Pagination transformsConfig `config:"pagination"` diff --git a/x-pack/filebeat/input/httpjson/encoding.go b/x-pack/filebeat/input/httpjson/encoding.go index cc0a59dbc99..611894b0543 100644 --- a/x-pack/filebeat/input/httpjson/encoding.go +++ b/x-pack/filebeat/input/httpjson/encoding.go @@ -13,6 +13,8 @@ import ( "io" "net/http" + "github.com/elastic/mito/lib/xml" + "github.com/elastic/elastic-agent-libs/logp" ) @@ -101,6 +103,11 @@ func registerDecoders() { log.Debugf("registering decoder 'application/zip': returned error: %#v", registerDecoder("application/zip", decodeAsZip)) + + log.Debugf("registering decoder 'application/xml': returned error: %#v", + registerDecoder("application/xml", decodeAsXML)) + log.Debugf("registering decoder 'text/xml': returned error: %#v", + registerDecoder("text/xml; charset=utf-8", decodeAsXML)) } func encodeAsJSON(trReq transformable) ([]byte, error) { @@ -217,3 +224,13 @@ func decodeAsZip(p []byte, dst *response) error { return nil } + +func decodeAsXML(p []byte, dst *response) error { + cdata, body, err := xml.Unmarshal(bytes.NewReader(p), dst.xmlDetails) + if err != nil { + return err + } + dst.body = body + dst.header["XML-CDATA"] = []string{cdata} + return nil +} diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index fc709c0b70f..408380b92a3 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -20,6 +20,8 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" + "github.com/elastic/mito/lib/xml" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/beat" @@ -128,8 +130,16 @@ func run( log.Errorf("Error while creating requestFactory: %v", err) return err } + var xmlDetails map[string]xml.Detail + if config.Response.XSD != "" { + xmlDetails, err = xml.Details([]byte(config.Response.XSD)) + if err != nil { + log.Errorf("error while collecting xml decoder type hints: %v", err) + return err + } + } pagination := newPagination(config, httpClient, log) - responseProcessor := newResponseProcessor(config, pagination, log) + responseProcessor := newResponseProcessor(config, pagination, xmlDetails, log) requester := newRequester(httpClient, requestFactory, responseProcessor, log) trCtx := emptyTransformContext() diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 3360dd0672f..18470520747 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -23,6 +23,7 @@ import ( beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) func TestInput(t *testing.T) { @@ -1176,6 +1177,104 @@ func TestInput(t *testing.T) { `{"space":{"world":"moon"}}`, }, }, + { + name: "Test simple XML decode", + setupServer: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + registerDecoders() + registerRequestTransforms() + r := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + const text = ` + + Ástríðr Ragnar +
+ Joord Lennart + Sydøstlige Gruppe +
Beekplantsoen 594, 2 hoog, 6849 IG
+ Boekend + Netherlands +
+ + Egil's Saga + Free Sample + 1 + 99.95 + FALSE + + +` + io.ReadAll(r.Body) + r.Body.Close() + w.Write([]byte(text)) + }) + server := httptest.NewServer(r) + t.Cleanup(func() { registeredTransforms = newRegistry() }) + config["request.url"] = server.URL + t.Cleanup(server.Close) + }, + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": http.MethodGet, + "response.xsd": ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +`, + }, + handler: defaultHandler(http.MethodGet, "", ""), + expected: []string{mapstr.M{ + "order": map[string]interface{}{ + "address": map[string]interface{}{ + "address": "Beekplantsoen 594, 2 hoog, 6849 IG", + "city": "Boekend", + "company": "Sydøstlige Gruppe", + "country": "Netherlands", + "name": "Joord Lennart", + }, + "item": []interface{}{ + map[string]interface{}{ + "cost": 99.95, + "name": "Egil's Saga", + "note": "Free Sample", + "number": 1, + "sent": false, + }, + }, + "noNamespaceSchemaLocation": "sales.xsd", + "orderid": "56733", + "sender": "Ástríðr Ragnar", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", + }, + }.String()}, + }, } for _, testCase := range testCases { diff --git a/x-pack/filebeat/input/httpjson/pagination.go b/x-pack/filebeat/input/httpjson/pagination.go index c8b8d128d84..ddaf5e8c670 100644 --- a/x-pack/filebeat/input/httpjson/pagination.go +++ b/x-pack/filebeat/input/httpjson/pagination.go @@ -11,6 +11,8 @@ import ( "net/http" "net/url" + "github.com/elastic/mito/lib/xml" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -90,18 +92,21 @@ type pageIterator struct { resp *http.Response + xmlDetails map[string]xml.Detail + isFirst bool done bool n int64 } -func (p *pagination) newPageIterator(stdCtx context.Context, trCtx *transformContext, resp *http.Response) *pageIterator { +func (p *pagination) newPageIterator(stdCtx context.Context, trCtx *transformContext, resp *http.Response, xmlDetails map[string]xml.Detail) *pageIterator { return &pageIterator{ pagination: p, stdCtx: stdCtx, trCtx: trCtx, resp: resp, + xmlDetails: xmlDetails, isFirst: true, } } @@ -179,6 +184,7 @@ func (iter *pageIterator) getPage() (*response, error) { if iter.pagination.decoder != nil { err = iter.pagination.decoder(bodyBytes, &r) } else { + r.xmlDetails = iter.xmlDetails err = decode(iter.resp.Header.Get("Content-Type"), bodyBytes, &r) } if err != nil { diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 9832275a326..ddba3216650 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -19,6 +19,7 @@ import ( inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/mito/lib/xml" ) const requestNamespace = "request" @@ -127,6 +128,15 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] rf.user = config.Auth.Basic.User rf.password = config.Auth.Basic.Password } + var xmlDetails map[string]xml.Detail + if config.Response.XSD != "" { + var err error + xmlDetails, err = xml.Details([]byte(config.Response.XSD)) + if err != nil { + log.Errorf("error while collecting xml decoder type hints: %v", err) + return nil, err + } + } rfs = append(rfs, rf) for _, ch := range config.Chain { var rf *requestFactory @@ -142,7 +152,7 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] rf.user = ch.Step.Auth.Basic.User rf.password = ch.Step.Auth.Basic.Password } - responseProcessor := newChainResponseProcessor(ch, httpClient, log) + responseProcessor := newChainResponseProcessor(ch, httpClient, xmlDetails, log) rf = &requestFactory{ url: *ch.Step.Request.URL.URL, method: ch.Step.Request.Method, @@ -168,7 +178,7 @@ func newRequestFactory(ctx context.Context, config config, log *logp.Logger) ([] rf.user = ch.While.Auth.Basic.User rf.password = ch.While.Auth.Basic.Password } - responseProcessor := newChainResponseProcessor(ch, httpClient, log) + responseProcessor := newChainResponseProcessor(ch, httpClient, xmlDetails, log) rf = &requestFactory{ url: *ch.While.Request.URL.URL, method: ch.While.Request.Method, diff --git a/x-pack/filebeat/input/httpjson/request_test.go b/x-pack/filebeat/input/httpjson/request_test.go index 67c270d24fe..f93eff7b7ad 100644 --- a/x-pack/filebeat/input/httpjson/request_test.go +++ b/x-pack/filebeat/input/httpjson/request_test.go @@ -65,7 +65,7 @@ func TestCtxAfterDoRequest(t *testing.T) { requestFactory, err := newRequestFactory(ctx, config, log) assert.NoError(t, err) pagination := newPagination(config, client, log) - responseProcessor := newResponseProcessor(config, pagination, log) + responseProcessor := newResponseProcessor(config, pagination, nil, log) requester := newRequester(client, requestFactory, responseProcessor, log) diff --git a/x-pack/filebeat/input/httpjson/response.go b/x-pack/filebeat/input/httpjson/response.go index 700e3b3901e..7555fcae63d 100644 --- a/x-pack/filebeat/input/httpjson/response.go +++ b/x-pack/filebeat/input/httpjson/response.go @@ -9,6 +9,8 @@ import ( "net/http" "net/url" + "github.com/elastic/mito/lib/xml" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -22,10 +24,11 @@ func registerResponseTransforms() { } type response struct { - page int64 - url url.URL - header http.Header - body interface{} + page int64 + url url.URL + header http.Header + xmlDetails map[string]xml.Detail + body interface{} } func (resp *response) clone() *response { @@ -54,13 +57,15 @@ type responseProcessor struct { transforms []basicTransform split *split pagination *pagination + xmlDetails map[string]xml.Detail } -func newResponseProcessor(config config, pagination *pagination, log *logp.Logger) []*responseProcessor { +func newResponseProcessor(config config, pagination *pagination, xmlDetails map[string]xml.Detail, log *logp.Logger) []*responseProcessor { rps := make([]*responseProcessor, 0, len(config.Chain)+1) rp := &responseProcessor{ pagination: pagination, + xmlDetails: xmlDetails, log: log, } if config.Response == nil { @@ -78,6 +83,7 @@ func newResponseProcessor(config config, pagination *pagination, log *logp.Logge for _, ch := range config.Chain { rp := &responseProcessor{ pagination: pagination, + xmlDetails: xmlDetails, log: log, } // chain calls responseProcessor object @@ -95,11 +101,12 @@ func newResponseProcessor(config config, pagination *pagination, log *logp.Logge return rps } -func newChainResponseProcessor(config chainConfig, httpClient *httpClient, log *logp.Logger) *responseProcessor { +func newChainResponseProcessor(config chainConfig, httpClient *httpClient, xmlDetails map[string]xml.Detail, log *logp.Logger) *responseProcessor { pagination := &pagination{httpClient: httpClient, log: log} rp := &responseProcessor{ pagination: pagination, + xmlDetails: xmlDetails, log: log, } if config.Step != nil { @@ -137,7 +144,7 @@ func (rp *responseProcessor) startProcessing(stdCtx context.Context, trCtx *tran defer close(ch) for i, httpResp := range resps { - iter := rp.pagination.newPageIterator(stdCtx, trCtx, httpResp) + iter := rp.pagination.newPageIterator(stdCtx, trCtx, httpResp, rp.xmlDetails) for { page, hasNext, err := iter.next() if err != nil {