diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 88c9c2099f8..f69f030f54e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -209,6 +209,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Reduce HTTPJSON metrics allocations. {pull}36282[36282] - Add support for a simplified input configuraton when running under Elastic-Agent {pull}36390[36390] - Make HTTPJSON response body decoding errors more informative. {pull}36481[36481] +- Allow fine-grained control of entity analytics API requests for Okta provider. {issue}36440[36440] {pull}36492[36492] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc index 8664dd23fa4..2957b5b7b43 100644 --- a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc @@ -545,6 +545,7 @@ Example configuration: enabled: true id: okta-1 provider: okta + dataset: "all" sync_interval: "12h" update_interval: "30m" okta_domain: "OKTA_DOMAIN" @@ -570,6 +571,14 @@ Whether the input should collect device and device-associated user details from the Okta API. Device details must be activated on the Okta account for this option. +[float] +===== `dataset` + +The datasets to collect from the API. This can be one of "all", "users" or "devices", +or may be left empty for the default behavior which is to collect all entities. +When the `dataset` is set to "devices", some user entity data is collected in order +to populate the registered users and registered owner fields for each device. + [float] ===== `sync_interval` diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go index 44cac308075..e344b56478f 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go @@ -6,6 +6,7 @@ package okta import ( "errors" + "strings" "time" "github.com/elastic/elastic-agent-libs/transport/httpcommon" @@ -41,10 +42,10 @@ type conf struct { OktaDomain string `config:"okta_domain" validate:"required"` OktaToken string `config:"okta_token" validate:"required"` - // WantDevices indicates that device details - // should be collected. This is optional as - // the devices API is not necessarily activated. - WantDevices bool `config:"collect_device_details"` + // Dataset specifies the datasets to collect from + // the API. It can be ""/"all", "users", or + // "devices". + Dataset string `config:"dataset"` // SyncInterval is the time between full // synchronisation operations. @@ -159,7 +160,11 @@ func (c *conf) Validate() error { return errInvalidUpdateInterval case c.SyncInterval <= c.UpdateInterval: return errSyncBeforeUpdate - default: + } + switch strings.ToLower(c.Dataset) { + case "", "all", "users", "devices": return nil + default: + return errors.New("dataset must be 'all', 'users', 'devices' or empty") } } diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 2aeb57e2f6b..4aff3cd3e59 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "net/url" + "strings" "time" "github.com/hashicorp/go-retryablehttp" @@ -338,6 +339,13 @@ func (p *oktaInput) runIncrementalUpdate(inputCtx v2.Context, store *kvstore.Sto // any existing deltaLink will be ignored, forcing a full synchronization from Okta. // Returns a set of modified users by ID. func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSync bool) ([]*User, error) { + switch strings.ToLower(p.cfg.Dataset) { + case "", "all", "users": + default: + p.logger.Debugf("Skipping user collection from API: dataset=%s", p.cfg.Dataset) + return nil, nil + } + var ( query url.Values err error @@ -418,7 +426,10 @@ func (p *oktaInput) doFetchUsers(ctx context.Context, state *stateStore, fullSyn // synchronization from Okta. // Returns a set of modified devices by ID. func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullSync bool) ([]*Device, error) { - if !p.cfg.WantDevices { + switch strings.ToLower(p.cfg.Dataset) { + case "", "all", "devices": + default: + p.logger.Debugf("Skipping device collection from API: dataset=%s", p.cfg.Dataset) return nil, nil } @@ -482,7 +493,9 @@ func (p *oktaInput) doFetchDevices(ctx context.Context, state *stateStore, fullS // Users are not stored in the state as they are in doFetchUsers. We expect // them to already have been discovered/stored from that call and are stored - // associated with the device undecorated with discovery state. + // associated with the device undecorated with discovery state. Or, if the + // the dataset is set to "devices", then we have been asked not to care about + // this detail. batch[i].Users = append(batch[i].Users, users...) next, err := okta.Next(h) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go index d10b81061c9..1286cc24689 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go @@ -22,156 +22,186 @@ import ( ) func TestOktaDoFetch(t *testing.T) { - dbFilename := "TestOktaDoFetch.db" - store := testSetupStore(t, dbFilename) - t.Cleanup(func() { - testCleanupStore(store, dbFilename) - }) - - const ( - window = time.Minute - key = "token" - users = `[{"id":"USERID","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/USERID"}}}]` - devices = `[{"id":"DEVICEID","status":"STATUS","created":"2019-10-02T18:03:07.000Z","lastUpdated":"2019-10-02T18:03:07.000Z","profile":{"displayName":"Example Device name 1","platform":"WINDOWS","serialNumber":"XXDDRFCFRGF3M8MD6D","sid":"S-1-11-111","registered":true,"secureHardwarePresent":false,"diskEncryptionType":"ALL_INTERNAL_VOLUMES"},"resourceType":"UDDevice","resourceDisplayName":{"value":"Example Device name 1","sensitive":false},"resourceAlternateId":null,"resourceId":"DEVICEID","_links":{"activate":{"href":"https://localhost/api/v1/devices/DEVICEID/lifecycle/activate","hints":{"allow":["POST"]}},"self":{"href":"https://localhost/api/v1/devices/DEVICEID","hints":{"allow":["GET","PATCH","PUT"]}},"users":{"href":"https://localhost/api/v1/devices/DEVICEID/users","hints":{"allow":["GET"]}}}}]` - ) - - data := map[string]string{ - "users": users, - "devices": devices, + tests := []struct { + dataset string + wantUsers bool + wantDevices bool + }{ + {dataset: "", wantUsers: true, wantDevices: true}, + {dataset: "all", wantUsers: true, wantDevices: true}, + {dataset: "users", wantUsers: true, wantDevices: false}, + {dataset: "devices", wantUsers: false, wantDevices: true}, } - var wantUsers []User - err := json.Unmarshal([]byte(users), &wantUsers) - if err != nil { - t.Fatalf("failed to unmarshal user data: %v", err) - } - var wantDevices []Device - err = json.Unmarshal([]byte(users), &wantDevices) - if err != nil { - t.Fatalf("failed to unmarshal device data: %v", err) - } - - wantStates := make(map[string]State) - - // Set the number of repeats. - const repeats = 3 - var n int - ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Leave 49 remaining, reset in one minute. - w.Header().Add("x-rate-limit-limit", "50") - w.Header().Add("x-rate-limit-remaining", "49") - w.Header().Add("x-rate-limit-reset", fmt.Sprint(time.Now().Add(time.Minute).Unix())) - - if strings.HasPrefix(r.URL.Path, "/api/v1/device") && strings.HasSuffix(r.URL.Path, "users") { - // Give one user if this is a get device users request. - fmt.Fprintln(w, data["users"]) - return - } - - base := path.Base(r.URL.Path) - - // Set next link if we can still repeat. - n++ - if n < repeats { - w.Header().Add("link", fmt.Sprintf(`; rel="next"`, base)) - } - - prefix := strings.TrimRight(base, "s") // endpoints are plural. - id := fmt.Sprintf("%sid%d", prefix, n) - - // Store expected states. The State values are all Discovered - // unless the user is deleted since they are all first appearance. - states := []string{ - "ACTIVE", - "RECOVERY", - "DEPROVISIONED", - } - status := states[n%len(states)] - state := Discovered - if status == "DEPROVISIONED" { - state = Deleted - } - wantStates[id] = state - - replacer := strings.NewReplacer( - strings.ToUpper(prefix+"id"), id, - "STATUS", status, - ) - fmt.Fprintln(w, replacer.Replace(data[base])) - })) - defer ts.Close() - - u, err := url.Parse(ts.URL) - if err != nil { - t.Errorf("failed to parse server URL: %v", err) - } - a := oktaInput{ - cfg: conf{ - OktaDomain: u.Host, - OktaToken: key, - WantDevices: true, - }, - client: ts.Client(), - lim: rate.NewLimiter(1, 1), - logger: logp.L(), - } - - ss, err := newStateStore(store) - if err != nil { - t.Fatalf("unexpected error making state store: %v", err) - } - defer ss.close(false) - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - t.Run("users", func(t *testing.T) { - n = 0 - - got, err := a.doFetchUsers(ctx, ss, false) - if err != nil { - t.Fatalf("unexpected error from doFetch: %v", err) - } + for _, test := range tests { + t.Run(test.dataset, func(t *testing.T) { + suffix := test.dataset + if suffix != "" { + suffix = "_" + suffix + } + dbFilename := fmt.Sprintf("TestOktaDoFetch%s.db", suffix) + store := testSetupStore(t, dbFilename) + t.Cleanup(func() { + testCleanupStore(store, dbFilename) + }) + + const ( + window = time.Minute + key = "token" + users = `[{"id":"USERID","status":"STATUS","created":"2023-05-14T13:37:20.000Z","activated":null,"statusChanged":"2023-05-15T01:50:30.000Z","lastLogin":"2023-05-15T01:59:20.000Z","lastUpdated":"2023-05-15T01:50:32.000Z","passwordChanged":"2023-05-15T01:50:32.000Z","type":{"id":"typeid"},"profile":{"firstName":"name","lastName":"surname","mobilePhone":null,"secondEmail":null,"login":"name.surname@example.com","email":"name.surname@example.com"},"credentials":{"password":{"value":"secret"},"emails":[{"value":"name.surname@example.com","status":"VERIFIED","type":"PRIMARY"}],"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://localhost/api/v1/users/USERID"}}}]` + devices = `[{"id":"DEVICEID","status":"STATUS","created":"2019-10-02T18:03:07.000Z","lastUpdated":"2019-10-02T18:03:07.000Z","profile":{"displayName":"Example Device name 1","platform":"WINDOWS","serialNumber":"XXDDRFCFRGF3M8MD6D","sid":"S-1-11-111","registered":true,"secureHardwarePresent":false,"diskEncryptionType":"ALL_INTERNAL_VOLUMES"},"resourceType":"UDDevice","resourceDisplayName":{"value":"Example Device name 1","sensitive":false},"resourceAlternateId":null,"resourceId":"DEVICEID","_links":{"activate":{"href":"https://localhost/api/v1/devices/DEVICEID/lifecycle/activate","hints":{"allow":["POST"]}},"self":{"href":"https://localhost/api/v1/devices/DEVICEID","hints":{"allow":["GET","PATCH","PUT"]}},"users":{"href":"https://localhost/api/v1/devices/DEVICEID/users","hints":{"allow":["GET"]}}}}]` + ) + + data := map[string]string{ + "users": users, + "devices": devices, + } - if len(got) != repeats { - t.Errorf("unexpected number of results: got:%d want:%d", len(got), repeats) - } - for i, g := range got { - if wantID := fmt.Sprintf("userid%d", i+1); g.ID != wantID { - t.Errorf("unexpected user ID for user %d: got:%s want:%s", i, g.ID, wantID) + var wantUsers []User + if test.wantUsers { + err := json.Unmarshal([]byte(users), &wantUsers) + if err != nil { + t.Fatalf("failed to unmarshal user data: %v", err) + } } - if g.State != wantStates[g.ID] { - t.Errorf("unexpected user ID for user %s: got:%s want:%s", g.ID, g.State, wantStates[g.ID]) + var wantDevices []Device + if test.wantDevices { + err := json.Unmarshal([]byte(users), &wantDevices) + if err != nil { + t.Fatalf("failed to unmarshal device data: %v", err) + } } - } - }) - - t.Run("devices", func(t *testing.T) { - n = 0 - - got, err := a.doFetchDevices(ctx, ss, false) - if err != nil { - t.Fatalf("unexpected error from doFetch: %v", err) - } - - if len(got) != repeats { - t.Errorf("unexpected number of results: got:%d want:%d", len(got), repeats) - } - for i, g := range got { - if wantID := fmt.Sprintf("deviceid%d", i+1); g.ID != wantID { - t.Errorf("unexpected device ID for device %d: got:%s want:%s", i, g.ID, wantID) + + wantStates := make(map[string]State) + + // Set the number of repeats. + const repeats = 3 + var n int + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Leave 49 remaining, reset in one minute. + w.Header().Add("x-rate-limit-limit", "50") + w.Header().Add("x-rate-limit-remaining", "49") + w.Header().Add("x-rate-limit-reset", fmt.Sprint(time.Now().Add(time.Minute).Unix())) + + if strings.HasPrefix(r.URL.Path, "/api/v1/device") && strings.HasSuffix(r.URL.Path, "users") { + // Give one user if this is a get device users request. + fmt.Fprintln(w, data["users"]) + return + } + + base := path.Base(r.URL.Path) + + // Set next link if we can still repeat. + n++ + if n < repeats { + w.Header().Add("link", fmt.Sprintf(`; rel="next"`, base)) + } + + prefix := strings.TrimRight(base, "s") // endpoints are plural. + id := fmt.Sprintf("%sid%d", prefix, n) + + // Store expected states. The State values are all Discovered + // unless the user is deleted since they are all first appearance. + states := []string{ + "ACTIVE", + "RECOVERY", + "DEPROVISIONED", + } + status := states[n%len(states)] + state := Discovered + if status == "DEPROVISIONED" { + state = Deleted + } + wantStates[id] = state + + replacer := strings.NewReplacer( + strings.ToUpper(prefix+"id"), id, + "STATUS", status, + ) + fmt.Fprintln(w, replacer.Replace(data[base])) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Errorf("failed to parse server URL: %v", err) } - if g.State != wantStates[g.ID] { - t.Errorf("unexpected device ID for device %s: got:%s want:%s", g.ID, g.State, wantStates[g.ID]) + a := oktaInput{ + cfg: conf{ + OktaDomain: u.Host, + OktaToken: key, + Dataset: test.dataset, + }, + client: ts.Client(), + lim: rate.NewLimiter(1, 1), + logger: logp.L(), } - if g.Users == nil { - t.Errorf("expected users for device %s", g.ID) + + ss, err := newStateStore(store) + if err != nil { + t.Fatalf("unexpected error making state store: %v", err) } - } + defer ss.close(false) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + t.Run("users", func(t *testing.T) { + n = 0 + + got, err := a.doFetchUsers(ctx, ss, false) + if err != nil { + t.Fatalf("unexpected error from doFetch: %v", err) + } + + if len(got) != wantCount(repeats, test.wantUsers) { + t.Errorf("unexpected number of results: got:%d want:%d", len(got), wantCount(repeats, test.wantUsers)) + } + for i, g := range got { + if wantID := fmt.Sprintf("userid%d", i+1); g.ID != wantID { + t.Errorf("unexpected user ID for user %d: got:%s want:%s", i, g.ID, wantID) + } + if g.State != wantStates[g.ID] { + t.Errorf("unexpected user state for user %s: got:%s want:%s", g.ID, g.State, wantStates[g.ID]) + } + } + }) + + t.Run("devices", func(t *testing.T) { + n = 0 + + got, err := a.doFetchDevices(ctx, ss, false) + if err != nil { + t.Fatalf("unexpected error from doFetch: %v", err) + } + + if len(got) != wantCount(repeats, test.wantDevices) { + t.Errorf("unexpected number of results: got:%d want:%d", len(got), wantCount(repeats, test.wantDevices)) + } + for i, g := range got { + if wantID := fmt.Sprintf("deviceid%d", i+1); g.ID != wantID { + t.Errorf("unexpected device ID for device %d: got:%s want:%s", i, g.ID, wantID) + } + if g.State != wantStates[g.ID] { + t.Errorf("unexpected device state for device %s: got:%s want:%s", g.ID, g.State, wantStates[g.ID]) + } + if g.Users == nil { + t.Errorf("expected users for device %s", g.ID) + } + } + + if t.Failed() { + b, _ := json.MarshalIndent(got, "", "\t") + t.Logf("document:\n%s", b) + } + }) + }) + } +} - if t.Failed() { - b, _ := json.MarshalIndent(got, "", "\t") - t.Logf("document:\n%s", b) - } - }) +func wantCount(n int, want bool) int { + if !want { + return 0 + } + return n }