Packetbeat: Handle ports and IPv6 in HTTP Host header (#14215)

The HTTP parser in Packetbeat wasn't properly handling the hostname
and port from "Host:" header, generating wrong "destination.domain" and
"url.full" fields.

CHANGELOG.next.asciidoc

@@ -242,6 +242,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improved debug logging efficiency in PGQSL module. {issue}12150[12150]
 - Limit memory usage of Redis replication sessions. {issue}12657[12657]
 - Fix parsing the extended RCODE in the DNS parser. {pull}12805[12805]
+- Fix parsing of the HTTP host header when it contains a port or an IPv6 address. {pull}14215[14215]
 
 *Winlogbeat*
 

packetbeat/protos/http/event.go

@@ -21,6 +21,7 @@ import (
 	"net"
 	"net/url"
 	"strconv"
+	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/ecs/code/go/ecs"
@@ -95,6 +96,8 @@ func synthesizeFullURL(u *ecs.Url, port int64) string {
 	host := u.Domain
 	if port != 80 {
 		host = net.JoinHostPort(u.Domain, strconv.Itoa(int(u.Port)))
+	} else if strings.IndexByte(u.Domain, ':') != -1 {
+		host = "[" + u.Domain + "]"
 	}
 
 	urlBuilder := url.URL{

packetbeat/protos/http/http.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -532,11 +533,16 @@ func (http *httpPlugin) newTransaction(requ, resp *message) beat.Event {
 			logp.Warn("Fail to parse HTTP parameters: %v", err)
 		}
 
-		host := string(requ.host)
 		pbf.Source.Bytes = int64(requ.size)
+		host, port := extractHostHeader(string(requ.host))
 		if net.ParseIP(host) == nil {
 			pbf.Destination.Domain = host
 		}
+		if port == 0 {
+			port = int(pbf.Destination.Port)
+		} else if port != int(pbf.Destination.Port) {
+			requ.notes = append(requ.notes, "Host header port number mismatch")
+		}
 		pbf.Event.Start = requ.ts
 		pbf.Network.ForwardedIP = string(requ.realIP)
 		pbf.Error.Message = requ.notes
@@ -554,7 +560,7 @@ func (http *httpPlugin) newTransaction(requ, resp *message) beat.Event {
 		httpFields.RequestHeaders = http.collectHeaders(requ)
 
 		// url
-		u := newURL(host, int64(pbf.Destination.Port), path, params)
+		u := newURL(host, int64(port), path, params)
 		pb.MarshalStruct(evt.Fields, "url", u)
 
 		// user-agent
@@ -701,6 +707,23 @@ func parseCookieValue(raw string) string {
 	return raw
 }
 
+func extractHostHeader(header string) (host string, port int) {
+	if len(header) == 0 || net.ParseIP(header) != nil {
+		return header, port
+	}
+	// Split :port trailer
+	if pos := strings.LastIndexByte(header, ':'); pos != -1 {
+		if num, err := strconv.Atoi(header[pos+1:]); err == nil && num > 0 && num < 65536 {
+			header, port = header[:pos], num
+		}
+	}
+	// Remove square bracket boxing of IPv6 address.
+	if last := len(header) - 1; header[0] == '[' && header[last] == ']' && net.ParseIP(header[1:last]) != nil {
+		header = header[1:last]
+	}
+	return header, port
+}
+
 func (http *httpPlugin) hideHeaders(m *message) {
 	if !m.isRequest || !http.redactAuthorization {
 		return

packetbeat/protos/http/http_test.go

@@ -1659,6 +1659,124 @@ func TestHTTP_Decoding_disabled(t *testing.T) {
 	assert.Equal(t, deflateBody, body)
 }
 
+func TestHttpParser_hostHeader(t *testing.T) {
+	template := "HEAD /_cat/shards HTTP/1.1\r\n" +
+		"Host: %s\r\n" +
+		"\r\n"
+	var store eventStore
+	http := httpModForTests(&store)
+	for _, test := range []struct {
+		title, host string
+		port        uint16
+		expected    common.MapStr
+	}{
+		{
+			title: "domain alone",
+			host:  "elasticsearch",
+			expected: common.MapStr{
+				"destination.domain": "elasticsearch",
+				"url.full":           "http://elasticsearch/_cat/shards",
+			},
+		},
+		{
+			title: "domain with port",
+			port:  9200,
+			host:  "elasticsearch:9200",
+			expected: common.MapStr{
+				"destination.domain": "elasticsearch",
+				"url.full":           "http://elasticsearch:9200/_cat/shards",
+			},
+		},
+		{
+			title: "ipv4",
+			host:  "127.0.0.1",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://127.0.0.1/_cat/shards",
+			},
+		},
+		{
+			title: "ipv4 with port",
+			port:  9200,
+			host:  "127.0.0.1:9200",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://127.0.0.1:9200/_cat/shards",
+			},
+		},
+		{
+			title: "ipv6 unboxed",
+			host:  "fd00::42",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://[fd00::42]/_cat/shards",
+			},
+		},
+		{
+			title: "ipv6 boxed",
+			host:  "[fd00::42]",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://[fd00::42]/_cat/shards",
+			},
+		},
+		{
+			title: "ipv6 boxed with port",
+			port:  9200,
+			host:  "[::1]:9200",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://[::1]:9200/_cat/shards",
+			},
+		},
+		{
+			title: "non boxed ipv6",
+			// This one is now illegal but it seems at some point the RFC
+			// didn't enforce the brackets when the port was omitted.
+			host: "fd00::1234",
+			expected: common.MapStr{
+				"destination.domain": nil,
+				"url.full":           "http://[fd00::1234]/_cat/shards",
+			},
+		},
+		{
+			title: "non-matching port",
+			port:  80,
+			host:  "myhost:9200",
+			expected: common.MapStr{
+				"destination.domain": "myhost",
+				"url.full":           "http://myhost:9200/_cat/shards",
+				"error.message":      []string{"Unmatched request", "Host header port number mismatch"},
+			},
+		},
+	} {
+		t.Run(test.title, func(t *testing.T) {
+			request := fmt.Sprintf(template, test.host)
+			tcptuple := testCreateTCPTuple()
+			if test.port != 0 {
+				tcptuple.DstPort = test.port
+			}
+			packet := protos.Packet{Payload: []byte(request)}
+			private := protos.ProtocolData(&httpConnectionData{})
+			private = http.Parse(&packet, tcptuple, 1, private)
+			http.Expired(tcptuple, private)
+			trans := expectTransaction(t, &store)
+			if !assert.NotNil(t, trans) {
+				t.Fatal("nil transaction")
+			}
+			for field, expected := range test.expected {
+				actual, err := trans.GetValue(field)
+				assert.Equal(t, expected, actual, field)
+				if expected != nil {
+					assert.Nil(t, err, field)
+				} else {
+					assert.Equal(t, common.ErrKeyNotFound, err, field)
+				}
+			}
+		})
+	}
+}
+
 func benchmarkHTTPMessage(b *testing.B, data []byte) {
 	http := httpModForTests(nil)
 	parser := newParser(&http.parserConfig)