Skip to content

Commit

Permalink
[Filebeat] Fix CEF time parsing (#13579)
Browse files Browse the repository at this point in the history 
* Fix CEF time parsing

Time fields were not being parsed because of a missing switch case statement. This fixes that problem, adds a beta logging statement to the code, and updates the CEF field name mapping for `catdt` to `categoryDeviceType`.

* Use common.Time for JSON marshaling
  • Loading branch information
@andrewkroh
andrewkroh committed Sep 17, 2019
1 parent fdd9d25 commit edb7c3d
Show file tree
Hide file tree
Showing 5 changed files with 599 additions and 5 deletions.
4 changes: 4 additions & 0 deletions x-pack/filebeat/processors/decode_cef/cef/keys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,8 @@ var fullNameMapping = map[string]string{
"start": "startTime",
"proto": "transportProtocol",
"type": "type",

// This is an ArcSight categorization field that is commonly used, but its
// short name is not contained in the documentation used for the above list.
"catdt": "categoryDeviceType",
}
3 changes: 3 additions & 0 deletions x-pack/filebeat/processors/decode_cef/decode_cef.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/cfgwarn"
"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/libbeat/processors"
"github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef"
Expand Down Expand Up @@ -44,6 +45,8 @@ func New(cfg *common.Config) (processors.Processor, error) {
}

func newDecodeCEF(c config) (*processor, error) {
cfgwarn.Beta("The " + procName + " processor is a beta feature.")

log := logp.NewLogger(logName)
if c.ID != "" {
log = log.With("instance_id", c.ID)
Expand Down
14 changes: 9 additions & 5 deletions x-pack/filebeat/processors/decode_cef/keys.ecs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"time"

"github.com/pkg/errors"

"github.com/elastic/beats/libbeat/common"
)

type dataType uint8
Expand Down Expand Up @@ -370,8 +372,10 @@ func toType(value string, typ dataType) (interface{}, error) {
return toBoolean(value)
case IP:
return toIP(value)
case Timestamp:
return toTimestamp(value)
default:
panic("invalid data type")
panic(errors.Errorf("invalid data type: %v", typ))
}
}

Expand Down Expand Up @@ -425,9 +429,9 @@ var timeLayouts = []string{
"Jan _2 2006 15:04:05",
}

func toTimestamp(v string) (time.Time, error) {
func toTimestamp(v string) (common.Time, error) {
if unixMs, err := toLong(v); err == nil {
return time.Unix(0, unixMs*int64(time.Millisecond)), nil
return common.Time(time.Unix(0, unixMs*int64(time.Millisecond))), nil
}

for _, layout := range timeLayouts {
Expand All @@ -439,9 +443,9 @@ func toTimestamp(v string) (time.Time, error) {
ts = ts.AddDate(currentYear, 0, 0)
}

return ts, nil
return common.Time(ts), nil
}
}

return time.Time{}, errors.New("value is not a valid timestamp")
return common.Time(time.Time{}), errors.New("value is not a valid timestamp")
}
8 changes: 8 additions & 0 deletions x-pack/filebeat/processors/decode_cef/testdata/samples.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,11 @@ CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE_XFORM|6|src=10.217.253.78
CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE|6|src=10.217.253.78 spt=56116 method=GET request=http://vpx247.example.net/FFC/CreditCardMind.html msg= Maximum no. of potential credit card numbers seen cn1=653 cn2=610 cs1=pr_ffc cs2=PPE0 cs3=li8MdGfW49uG8tGdSV85ech41a0A000 cs4=ALERT cs5=2012 act=transformed
CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SIGNATURE_MATCH|6|src=10.217.253.78 spt=56687 method=GET request=http://vpx247.example.net/FFC/wwwboard/passwd.txt msg= Signature violation rule ID 807: web-cgi /wwwboard/passwd.txt access cn1=224 cn2=205 cs1=pr_ffc cs2=PPE0 cs3=POousP7CIMW5nwZ5Rs4nq5DND0sA000 cs4=ALERT cs5=2012 act=not blocked
CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Something awesome happened|very-high|eventId=3457 requestMethod=POST dlat=38.915 dlong=-77.511 proto=TCP rawEvent={"x": "y"} sourceServiceName=httpd destinationServiceName=chrome requestContext=application/json
CEF:0|Microsoft|DNS Trace Log||Response:A|Response|Unknown| eventId=12345678 type=1 start=1322004689000 art=1322022474516 rt=1322005087000 src=10.0.0.2 dhost=www.google.com request=(3)www(6)google(3)com(0) cnt=2 ahost=arcagt1 agt=10.2.3.4 atz=America/New York aid=NpLHzDMCABCBBTXAZqYDUA\=\= at=dns_tracelog_file dtz=America/New York requestUrlFileName=(3)www(6)google(3)com(0) _cefVer=0.1
CEF:0|Unix|Unix||arcsight:143:1|Started Session|Low| eventId=31 msg=Started Session 21 of user root categorySignificance=/Informational categoryBehavior=/Access/Start categoryDeviceGroup=/Operating System catdt=Operating System categoryOutcome=/Success categoryObject=/Host/Application/Service art=1500404470493 deviceSeverity=info act=Started rt=1500404461000 suser=root dhost=centos7 cs1=systemd cs2=daemon cs1Label=Module cs2Label=Facility cn1Label=File Descriptor ahost=centos7.as agt=10.2.3.4 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 10.0.0.0-10.255.255.255 amac=00-50-56-8E-C0-90 av=7.6.0.8009.0 atz=America/Argentina/Buenos_Aires at=syslog dvchost=centos7 dtz=America/Argentina/Buenos_Aires deviceFacility=daemon deviceProcessName=systemd _cefVer=0.1 aid=4SNQXV30BABCAIi+-ZH3gxT\=\=
CEF:0|Check Point|VPN-1 & FireWall-1||drop|drop|High| eventId=23985829654 mrt=1459367145678 proto=TCP customerID=124 customerURI=/XXX modelConfidence=0.6 relevance=high categorySignificance=/Informational/Warning categoryBehavior=/Access categoryDeviceGroup=/Firewall catdt=Firewall categoryOutcome=/Failure categoryObject=/Host/Application/Service modelConfidence=0 severity=5 relevance=10 assetCriticality=0 priority=High
CEF:0|CISCO|ASA||305012|Teardown dynamic UDP translation|Low| eventId=56265798504 mrt=1484092683471 proto=UDP categorySignificance=/Informational categoryBehavior=/Access/Stop categoryDeviceGroup=/Firewall catdt=Firewall categoryOutcome=/Success categoryObject=/Host/Application/Service modelConfidence=0 severity=4 relevance=10 assetCriticality=0 priority=4 art=1484096108163 deviceSeverity=6 rt=1484096094000 src=1.2.3.4 sourceZoneID=GqtK3G9YBABCadQ465CqVeW\=\= sourceZoneURI=/All Zones/GTR/GTR/GTR/GTR sourceTranslatedAddress=4.3.2.1 sourceTranslatedZoneID=P84KXXTYDFYYFwwHq40BQcd\=\= sourceTranslatedZoneURI=/All Zones/GTR/GTR Internet Primary spt=5260 sourceTranslatedPort=5260 cs5=dynamic cs6=0:00:00 c6a4=ffff:0:0:0:222:5555:ffff:5555 locality=1 cs1Label=ACL cs2Label=Unit cs3Label=TCP Flags cs4Label=Order cs5Label=Connection Type cs6Label=Duration cn1Label=ICMP Type cn2Label=ICMP Code cn3Label=DurationInSeconds c6a4Label=Agent IPv6 Address ahost=host.gtr.gtr agt=100.222.333.55 av=7.1.7.7602.0 atz=LA/la aid=4p9IZi1kBABCq5RFPFdJWYUw\=\= at=agent_ac dvchost=super dvc=111.111.111.99 deviceZoneID=K-fU33AAOGVdfFpYAT3UdQ\=\= deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 deviceAssetId=5Wa8hHVSDFBCc-t56wI7mTw\=\= dtz=LA/LA deviceInboundInterface=eth0 deviceOutboundInterface=eth1 eventAnnotationStageUpdateTime=1484097686473 eventAnnotationModificationTime=1484097686475 eventAnnotationAuditTrail=1,1484012146095,root,Queued,,,,\\n eventAnnotationVersion=1 eventAnnotationFlags=0 eventAnnotationEndTime=1484096094000 eventAnnotationManagerReceiptTime=1484097686471 originalAgentHostName=host originalAgentAddress=10.2.88.3 originalAgentZoneURI=/All Zones/GR/GR/GR originalAgentVersion=7.3.0.7885.0 originalAgentId=6q0sfHVcBABCcSDFvMpvc1w\=\= originalAgentType=syslog_file _cefVer=0.1 ad.arcSightEventPath=7q0sfHVcBABCcMZVvMSDFc1w\=\=
CEF:0|ArcSight|ArcSight|7.0.5.7132.1|agent:016|Device connection up|Low| eventId=1 msg=File Opened mrt=1410524600502 categorySignificance=/Normal categoryBehavior=/Access/Start categoryDeviceGroup=/Application catdt=Security Mangement categoryOutcome=/Success categoryObject=/Host/Application art=1410524502535 cat=/Agent/Connection/Device?Success deviceSeverity=Warning rt=1410524500502 fname=C:\\Documents and Settings\\XPMUser\\Desktop\\Logs\\NAT_Log fileType=Agent cs2=<Resource ID\="3Qg5paUgBABCAAwIZ-kC0dw\=\="/> cs2Label=Configuration Resource ahost=VirtualXP agt=192.168.131.65 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 av=7.0.5.7132.0 atz=Europe/Prague aid=3Pz6paUgBABCAAudQNx1w0w\=\= at=sdkrfilereader dvchost=VirtualXP dvc=192.168.131.65 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Europe/Prague _cefVer=0.1
CEF:0|ArcSight|ArcSight|7.0.5.7132.1|agent:030|Agent [NAT] type [sdkrfilereader] started|Low| eventId=2 mrt=1410524500493 categorySignificance=/Normal categoryBehavior=/Execute/Start categoryDeviceGroup=/Application catdt=Security Mangement categoryOutcome=/Success categoryObject=/Host/Application/Service art=1410624402535 cat=/Agent/Started deviceSeverity=Warning rt=1410543500432 fileType=Agent cs2=<Resource ID\="3Tg5paUgBABCAAwIZ-kC0dw\=\="/> cs2Label=Configuration Resource ahost=VirtualXP agt=192.168.1.56 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 av=7.0.5.7132.1 atz=Europe/Prague aid=4Pz6paUgBABCAAudQNx1w0w\=\= at=sdkrfilereader dvchost=VirtualXP dvc=192.168.0.65 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Europe/Prague _cefVer=0.1
CEF:0|ArcSight|ArcSight|7.0.5.7132.1|agent:044|File processing started|Low| eventId=6 mrt=1410524500502 catdt=Security Mangement art=1410524502535 cat=/LogFile/Processing/Started deviceSeverity=Warning rt=1410524500502 fname=C:\\Documents and Settings\\XPMUser\\Desktop\\Logs\\NAT_Log ahost=VirtualXP agt=192.168.131.65 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 av=7.0.5.7132.0 atz=Europe/Prague aid=3Pz6paUgBABCAAudQNx1w0w\=\= at=sdkrfilereader dvchost=VirtualXP dvc=192.168.131.65 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Europe/Prague _cefVer=0.1
CEF:0|ArcSight|ArcSight|7.0.5.7132.1|agent:031|Agent [NAT] type [sdkrfilereader] shutting down|Very-High| eventId=7 msg=Process Stopped by User mrt=1410524535833 categorySignificance=/Normal categoryBehavior=/Execute/Stop categoryDeviceGroup=/Application catdt=Security Mangement categoryOutcome=/Success categoryObject=/Host/Application/Service art=1410524535843 cat=/Agent/ShuttingDown deviceSeverity=Warning rt=1410524535833 fileType=Agent cs2=<Resource ID\="3Qg5paUgBABCAAwIZ-kC0dw\=\="/> cs2Label=Configuration Resource ahost=VirtualXP agt=192.168.131.65 agentZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 av=7.0.5.7132.0 atz=Europe/Prague aid=3Pz6paUgBABCAAudQNx1w0w\=\= at=sdkrfilereader dvchost=VirtualXP dvc=192.168.131.65 deviceZoneURI=/All Zones/ArcSight System/Private Address Space Zones/RFC1918: 192.168.0.0-192.168.255.255 dtz=Europe/Prague _cefVer=0.1
Loading

0 comments on commit edb7c3d

Please sign in to comment.