Add test event without authorization_info[]

elastic · Aug 6, 2020 · f9e1ed9 · f9e1ed9
1 parent 9a7bfce
commit f9e1ed9
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -233,6 +233,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
 - Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
 - Fix `okta` geoip lookup in pipeline for `destination.ip` {pull}20454[20454]
+- Fix mapping exception in the `googlecloud/audit` dataset pipeline. {issue}18465[18465] {pull}20465[20465]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
@@ -243,11 +243,13 @@ function Audit(keep_original_message) {
     // Rename nested fields.
     var renameNestedFields = function(evt) {
         var arr = evt.Get("googlecloud.audit.authorization_info");
-        for (var i = 0; i < arr.length; i++) {
-            if (arr[i].resourceAttributes) {
-                // Convert to snake_case.
-                arr[i].resource_attributes = arr[i].resourceAttributes;
-                delete arr[i].resourceAttributes;
+        if (Array.isArray(arr)) {
+            for (var i = 0; i < arr.length; i++) {
+                if (arr[i].resourceAttributes) {
+                    // Convert to snake_case.
+                    arr[i].resource_attributes = arr[i].resourceAttributes;
+                    delete arr[i].resourceAttributes;
+                }
             }
         }
     };
@@ -265,17 +267,16 @@ function Audit(keep_original_message) {
         // Try to use authorization_info.granted when there was no status code.
         if (evt.Get("googlecloud.audit.status.code") == null) {
             var authorization_info = evt.Get("googlecloud.audit.authorization_info");
-            if (authorization_info.length === 1) {
-                if (authorization_info[0].granted == null) {
-                    evt.Put("event.outcome", "unknown");
-                } else if (authorization_info[0].granted === true) {
+            if (Array.isArray(authorization_info) && authorization_info.length === 1) {
+                if (authorization_info[0].granted === true) {
                     evt.Put("event.outcome", "success");
-                } else {
+                } else if (authorization_info[0].granted === false) {
                     evt.Put("event.outcome", "failure");
                 }
-            } else {
-                evt.Put("event.outcome", "unknown");
+                return
             }
+
+            evt.Put("event.outcome", "unknown");
             return;
         }
 

diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log
@@ -4,3 +4,4 @@
 {"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
 {"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"}
 {"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"}
+{"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"2.3.4.5","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"}
diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json
@@ -297,5 +297,51 @@
         "user_agent.os.full": "Mac OS X 19.6.0",
         "user_agent.os.name": "Mac OS X",
         "user_agent.os.version": "19.6.0"
+    },
+    {
+        "@timestamp": "2020-08-05T16:56:40.428Z",
+        "cloud.instance.id": "590261181",
+        "cloud.project.id": "foo",
+        "event.action": "beta.compute.instances.stop",
+        "event.dataset": "googlecloud.audit",
+        "event.id": "-c7ctxmd2zab",
+        "event.kind": "event",
+        "event.module": "googlecloud",
+        "event.outcome": "unknown",
+        "fileset.name": "audit",
+        "googlecloud.audit.authentication_info.principal_email": "user@mycompany.com",
+        "googlecloud.audit.method_name": "beta.compute.instances.stop",
+        "googlecloud.audit.request.proto_name": "type.googleapis.com/compute.instances.stop",
+        "googlecloud.audit.request_metadata.caller_ip": "2.3.4.5",
+        "googlecloud.audit.request_metadata.caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)",
+        "googlecloud.audit.resource_name": "projects/foo/zones/us-central1-a/instances/win10-test",
+        "googlecloud.audit.service_name": "compute.googleapis.com",
+        "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog",
+        "input.type": "log",
+        "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity",
+        "log.offset": 9946,
+        "service.name": "compute.googleapis.com",
+        "service.type": "googlecloud",
+        "source.as.number": 3215,
+        "source.as.organization.name": "Orange",
+        "source.geo.city_name": "Clermont-Ferrand",
+        "source.geo.continent_name": "Europe",
+        "source.geo.country_iso_code": "FR",
+        "source.geo.location.lat": 45.7838,
+        "source.geo.location.lon": 3.0966,
+        "source.geo.region_iso_code": "FR-63",
+        "source.geo.region_name": "Puy-de-D\u00f4me",
+        "source.ip": "2.3.4.5",
+        "tags": [
+            "forwarded"
+        ],
+        "user.email": "user@mycompany.com",
+        "user_agent.device.name": "Mac",
+        "user_agent.name": "Firefox",
+        "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)",
+        "user_agent.os.full": "Mac OS X 10.15",
+        "user_agent.os.name": "Mac OS X",
+        "user_agent.os.version": "10.15",
+        "user_agent.version": "79.0."
     }
 ]