Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added dns.question.top_level_domain and dns.question.subdomain ecs field #14578

Merged
merged 6 commits into from
Dec 20, 2019
Merged

Added dns.question.top_level_domain and dns.question.subdomain ecs field #14578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d1cc7f2
Added dns.question.top_level_domain and dns.question.subdomain ecs field
mbudge Nov 17, 2019
bef0f72
Add test data for TLD and subdomain
adriansr Nov 29, 2019
d13e230
Updated change log.
mbudge Dec 6, 2019
65d8806
Replaced strings.TrimRight with strings.TrimSuffix.
mbudge Dec 9, 2019
697d0e0
Added code to handle publicsuffix.EffectiveTLDPlusOne eTLD+1 error wi…
mbudge Dec 16, 2019
e779db1
Replaced strings.split with strings.IndexByte
mbudge Dec 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d

*Packetbeat*

- Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594]
- NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503]

Expand Down
20 changes: 19 additions & 1 deletion packetbeat/protos/dns/dns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,10 +497,28 @@ func addDNSToMapStr(m common.MapStr, dns *mkdns.Msg, authority bool, additional
m["question"] = qMapStr

eTLDPlusOne, err := publicsuffix.EffectiveTLDPlusOne(q.Name)
if err == nil {
if err == nil && eTLDPlusOne != "" {
eTLDPlusOne = strings.TrimRight(eTLDPlusOne, ".")

// etld_plus_one should be removed for 8.0.0.
qMapStr["etld_plus_one"] = eTLDPlusOne
qMapStr["registered_domain"] = eTLDPlusOne

if idx := strings.IndexByte(eTLDPlusOne, '.'); idx != -1 {
qMapStr["top_level_domain"] = eTLDPlusOne[idx+1:]
}

subdomain := strings.TrimRight(strings.TrimSuffix(q.Name, eTLDPlusOne), ".")
if subdomain != "" {
qMapStr["subdomain"] = subdomain
}
} else if strings.Count(q.Name, ".") == 1 {
// Handle publicsuffix.EffectiveTLDPlusOne eTLD+1 error with 1 dot in the domain.
s := strings.Split(q.Name, ".")
if len(s) == 2 && s[1] != "" {
qMapStr["top_level_domain"] = s[1]
}
qMapStr["registered_domain"] = q.Name
}
}

Expand Down
8 changes: 8 additions & 0 deletions packetbeat/protos/dns/dns_tcp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ var (
qType: "A",
qName: "elastic.co",
qEtld: "elastic.co",
qTLD: "co",
answers: []string{"54.201.204.244", "54.200.185.88"},
authorities: []string{"NS-835.AWSDNS-40.NET", "NS-1183.AWSDNS-19.ORG", "NS-2007.AWSDNS-58.CO.UK", "NS-66.AWSDNS-08.COM"},
request: []byte{
Expand Down Expand Up @@ -89,6 +90,7 @@ var (
qType: "AXFR",
qName: "etas.com",
qEtld: "etas.com",
qTLD: "com",
answers: []string{"training2003p", "training2003p", "1.1.1.1", "training2003p"},
request: []byte{
0x00, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x65,
Expand Down Expand Up @@ -120,6 +122,8 @@ var (
qType: "PTR",
qName: "131.252.30.192.in-addr.arpa",
qEtld: "192.in-addr.arpa",
qSubdomain: "131.252.30",
qTLD: "in-addr.arpa",
answers: []string{"github.com"},
authorities: []string{"ns1.p16.dynect.net", "ns3.p16.dynect.net", "ns4.p16.dynect.net", "ns2.p16.dynect.net"},
request: []byte{
Expand Down Expand Up @@ -152,6 +156,10 @@ var (
"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net",
qEtld: "sophosxl.net",
qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" +
"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac",
qTLD: "net",
request: []byte{
0x00, 0xed, 0x88, 0xc1, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33,
0x3f, 0x31, 0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34,
Expand Down
4 changes: 4 additions & 0 deletions packetbeat/protos/dns/dns_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type dnsTestMessage struct {
qType string
qName string
qEtld string
qSubdomain interface{}
qTLD interface{}
answers []string
authorities []string
additionals []string
Expand Down Expand Up @@ -263,6 +265,8 @@ func assertRequest(t testing.TB, m common.MapStr, q dnsTestMessage) {
assert.Equal(t, q.qClass, mapValue(t, m, "dns.question.class"))
assert.Equal(t, q.qType, mapValue(t, m, "dns.question.type"))
assert.Equal(t, q.qName, mapValue(t, m, "dns.question.name"))
assert.Equal(t, q.qTLD, mapValue(t, m, "dns.question.top_level_domain"))
assert.Equal(t, q.qSubdomain, mapValue(t, m, "dns.question.subdomain"))
assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.etld_plus_one"))
assert.Equal(t, q.qEtld, mapValue(t, m, "dns.question.registered_domain"))
}
Expand Down
46 changes: 28 additions & 18 deletions packetbeat/protos/dns/dns_udp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ var (
qType: "A",
qName: "elastic.co",
qEtld: "elastic.co",
qTLD: "co",
answers: []string{"54.148.130.30", "54.69.104.66"},
request: []byte{
0x21, 0x51, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x65, 0x6c, 0x61,
Expand All @@ -90,6 +91,7 @@ var (
qType: "IXFR",
qName: "etas.com",
qEtld: "etas.com",
qTLD: "com",
answers: []string{"training2003p", "training2003p", "training2003p", "training2003p", "1.1.1.100"},
request: []byte{
0x40, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x04, 0x65, 0x74, 0x61,
Expand Down Expand Up @@ -118,15 +120,17 @@ var (
}

githubPtr = dnsTestMessage{
id: 344,
opcode: "QUERY",
flags: []string{"rd", "ra"},
rcode: "NOERROR",
qClass: "IN",
qType: "PTR",
qName: "131.252.30.192.in-addr.arpa",
qEtld: "192.in-addr.arpa",
answers: []string{"github.com"},
id: 344,
opcode: "QUERY",
flags: []string{"rd", "ra"},
rcode: "NOERROR",
qClass: "IN",
qType: "PTR",
qName: "131.252.30.192.in-addr.arpa",
qEtld: "192.in-addr.arpa",
qSubdomain: "131.252.30",
qTLD: "in-addr.arpa",
answers: []string{"github.com"},
authorities: []string{"a.root-servers.net", "b.root-servers.net", "c.root-servers.net",
"d.root-servers.net", "e.root-servers.net", "f.root-servers.net", "g.root-servers.net",
"h.root-servers.net", "i.root-servers.net", "j.root-servers.net", "k.root-servers.net",
Expand Down Expand Up @@ -169,6 +173,10 @@ var (
"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac.sophosxl.net",
qEtld: "sophosxl.net",
qSubdomain: "3.1o19ss00s2s17s4qp375sp49r830n2n4n923s8839052s7p7768s53365226pp3.659p1r741os37393" +
"648s2348o762q1066q53rq5p4614r1q4781qpr16n809qp4.879o3o734q9sns005o3pp76q83.2q65qns3spns" +
"1081s5rn5sr74opqrqnpq6rn3ro5.i.00.mac",
qTLD: "net",
request: []byte{
0x20, 0x2e, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x33, 0x3f, 0x31,
0x6f, 0x31, 0x39, 0x73, 0x73, 0x30, 0x30, 0x73, 0x32, 0x73, 0x31, 0x37, 0x73, 0x34, 0x71, 0x70,
Expand Down Expand Up @@ -206,15 +214,17 @@ var (
}

ednsSecA = dnsTestMessage{
id: 20498,
opcode: "QUERY",
flags: []string{"rd", "ad", "ra"},
rcode: "NOERROR",
qClass: "IN",
qType: "A",
qName: "www.ietf.org",
qEtld: "ietf.org",
answers: []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="},
id: 20498,
opcode: "QUERY",
flags: []string{"rd", "ad", "ra"},
rcode: "NOERROR",
qClass: "IN",
qType: "A",
qName: "www.ietf.org",
qEtld: "ietf.org",
qTLD: "org",
qSubdomain: "www",
answers: []string{"64.170.98.30", "iDA8bJnrAEz3jgYnyFRm567a76qlv1V0CqxOSd/o9nvnN0GlZLaVoDmuXpaIaoypbGxwzwgK/LY6CV2k6SWKwicBmpENL26hwyjkFzPDW8kX3ibFhtfsOb8pYe7nBj326actp/7iG+DRuDmPnkYBja+wDYk61doTtkqZg57fn3iS97tjNPCC9C9knRAuDYUG+dVxalazSwYrpvY97dUC1H2spD0g4UdDyCbGA46mouZ4GPzNMewgf948qxrnU8pWPk3nQW5TgLVkGoWgco2owfLElBqf6rJ4LOswuhaw8IpTtmw3FsixxTLQvKOE5nftd1nMhQQd9CaHjoKNAUEz5Q=="},
request: []byte{
0x50, 0x12, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x77, 0x77, 0x77,
0x04, 0x69, 0x65, 0x74, 0x66, 0x03, 0x6f, 0x72, 0x67, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00,
Expand Down