From a074e4d0348d4ac7858573dd857a5df63f820198 Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Tue, 7 Jan 2020 11:47:24 +0000 Subject: [PATCH 1/5] Add redact_headers configuration option, which allows specific HTTP request headers to be redacted --- packetbeat/packetbeat.reference.yml | 5 ++++ packetbeat/protos/http/config.go | 1 + packetbeat/protos/http/http.go | 12 ++++++++++ packetbeat/protos/http/http_test.go | 37 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 042e53b6b83..86ceb94d6c4 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -200,6 +200,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/protos/http/config.go b/packetbeat/protos/http/config.go index 2197dbad088..14b3f55e359 100644 --- a/packetbeat/protos/http/config.go +++ b/packetbeat/protos/http/config.go @@ -36,6 +36,7 @@ type httpConfig struct { RedactAuthorization bool `config:"redact_authorization"` MaxMessageSize int `config:"max_message_size"` DecodeBody bool `config:"decode_body"` + RedactHeaders []string `config:"redact_headers"` } var ( diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 69821429c6c..dbc76badfd8 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -88,6 +88,7 @@ type httpPlugin struct { splitCookie bool hideKeywords []string redactAuthorization bool + redactHeaders []string maxMessageSize int mustDecodeBody bool @@ -147,6 +148,11 @@ func (http *httpPlugin) setFromConfig(config *httpConfig) { http.transactionTimeout = config.TransactionTimeout http.mustDecodeBody = config.DecodeBody + http.redactHeaders = make([]string, len(config.RedactHeaders)) + for i, header := range config.RedactHeaders { + http.redactHeaders[i] = strings.ToLower(header) + } + for _, list := range [][]string{config.IncludeBodyFor, config.IncludeRequestBodyFor} { http.parserConfig.includeRequestBodyFor = append(http.parserConfig.includeRequestBodyFor, list...) } @@ -725,6 +731,12 @@ func extractHostHeader(header string) (host string, port int) { } func (http *httpPlugin) hideHeaders(m *message) { + for _, header := range http.redactHeaders { + if _, exists := m.headers[header]; exists { + m.headers[header] = []byte("REDACTED") + } + } + if !m.isRequest || !http.redactAuthorization { return } diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go index 12e56feefed..f07af54008d 100644 --- a/packetbeat/protos/http/http_test.go +++ b/packetbeat/protos/http/http_test.go @@ -990,6 +990,43 @@ func TestHttpParser_RedactAuthorization_Proxy_raw(t *testing.T) { } } +func TestHttpParser_RedactHeaders(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("http", "httpdetailed")) + + http := httpModForTests(nil) + http.redactAuthorization = true + http.parserConfig.sendHeaders = true + http.parserConfig.sendAllHeaders = true + http.redactHeaders = []string{"header-to-redact", "should-not-exist"} + + data := []byte("POST /services/ObjectControl?ID=client0 HTTP/1.1\r\n" + + "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 2.0.50727.5472)\r\n" + + "Content-Type: text/xml; charset=utf-8\r\n" + + "SOAPAction: \"\"\r\n" + + "Header-To-Redact: sensitive-value\r\n" + + "Host: production.example.com\r\n" + + "Content-Length: 0\r\n" + + "Expect: 100-continue\r\n" + + "Accept-Encoding: gzip\r\n" + + "X-Forwarded-For: 10.216.89.132\r\n" + + "\r\n") + + st := &stream{data: data, message: new(message)} + + ok, _ := testParseStream(http, st, 0) + + http.hideHeaders(st.message) + + assert.True(t, ok) + var redactedString common.NetString = []byte("REDACTED") + var expectedAcceptEncoding common.NetString = []byte("gzip") + assert.Equal(t, redactedString, st.message.headers["header-to-redact"]) + assert.Equal(t, expectedAcceptEncoding, st.message.headers["accept-encoding"]) + + _, invalidHeaderExists := st.message.headers["should-not-exist"] + assert.False(t, invalidHeaderExists) +} + func Test_splitCookiesHeader(t *testing.T) { type io struct { Input string From c435a13e06e2d9e398791d0220291726102b9d81 Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Tue, 7 Jan 2020 16:48:44 +0000 Subject: [PATCH 2/5] ran mage fmt update --- packetbeat/_meta/beat.reference.yml | 5 +++++ packetbeat/tests/system/packetbeat.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packetbeat/_meta/beat.reference.yml b/packetbeat/_meta/beat.reference.yml index a379b1e4f54..237eac991fe 100644 --- a/packetbeat/_meta/beat.reference.yml +++ b/packetbeat/_meta/beat.reference.yml @@ -200,6 +200,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 24b59a32b9c..eba62b56bab 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -1,3 +1,5 @@ +from beat.beat import Proc +from beat.beat import TestCase import os import sys import subprocess @@ -5,8 +7,6 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system')) -from beat.beat import TestCase -from beat.beat import Proc TRANS_REQUIRED_FIELDS = ["@timestamp", "type", "status", "agent.type", "agent.hostname", "agent.version", From 314d0c225e66b0668fb6731c465f2327369548da Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Tue, 7 Jan 2020 18:18:12 +0000 Subject: [PATCH 3/5] Added asciidoc and changelog entries --- CHANGELOG.next.asciidoc | 1 + packetbeat/docs/packetbeat-options.asciidoc | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f2340dfbdd4..d1c5634ba28 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -82,6 +82,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503] +- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. *Winlogbeat* diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index f8767956726..b87193a093c 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -682,6 +682,12 @@ headers are placed under the `headers` dictionary in the resulting JSON. Instead of sending a white list of headers to Elasticsearch, you can send all headers by setting this option to true. The default is false. +===== `redact_headers` + +A list of headers to redact if present in the HTTP request. This will keep +the header field present, but will redact it's value to show the header's +presence. + ===== `include_body_for` The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under From b2db1848212ede541bdba153f41f44f02e053da7 Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Wed, 8 Jan 2020 10:13:36 +0000 Subject: [PATCH 4/5] revert 'mage fmt update' change to packetbeat test --- packetbeat/tests/system/packetbeat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index eba62b56bab..24b59a32b9c 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -1,5 +1,3 @@ -from beat.beat import Proc -from beat.beat import TestCase import os import sys import subprocess @@ -7,6 +5,8 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system')) +from beat.beat import TestCase +from beat.beat import Proc TRANS_REQUIRED_FIELDS = ["@timestamp", "type", "status", "agent.type", "agent.hostname", "agent.version", From 57b29279d0f12a81e171d047cbc2144e3c84d305 Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Wed, 8 Jan 2020 14:54:10 +0000 Subject: [PATCH 5/5] Update CHANGELOG.next.asciidoc Co-Authored-By: Adrian Serrano --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d1c5634ba28..03858f21cec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -82,7 +82,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503] -- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. +- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353] *Winlogbeat*