From 7e8e971813b53954803a2466d9f233cc5fd6c1ad Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 8 Jul 2020 12:29:47 +0200 Subject: [PATCH] Cherry-pick #19378 to 7.x: Cursor input skeleton (#19710) --- filebeat/input/v2/input-cursor/clean.go | 45 +++++ filebeat/input/v2/input-cursor/cursor.go | 43 +++++ filebeat/input/v2/input-cursor/doc.go | 58 +++++++ filebeat/input/v2/input-cursor/input.go | 84 +++++++++ filebeat/input/v2/input-cursor/manager.go | 106 +++++++++++ filebeat/input/v2/input-cursor/publish.go | 72 ++++++++ filebeat/input/v2/input-cursor/store.go | 203 ++++++++++++++++++++++ 7 files changed, 611 insertions(+) create mode 100644 filebeat/input/v2/input-cursor/clean.go create mode 100644 filebeat/input/v2/input-cursor/cursor.go create mode 100644 filebeat/input/v2/input-cursor/doc.go create mode 100644 filebeat/input/v2/input-cursor/input.go create mode 100644 filebeat/input/v2/input-cursor/manager.go create mode 100644 filebeat/input/v2/input-cursor/publish.go create mode 100644 filebeat/input/v2/input-cursor/store.go diff --git a/filebeat/input/v2/input-cursor/clean.go b/filebeat/input/v2/input-cursor/clean.go new file mode 100644 index 00000000000..e833a9eead0 --- /dev/null +++ b/filebeat/input/v2/input-cursor/clean.go @@ -0,0 +1,45 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/unison" +) + +// cleaner removes finished entries from the registry file. +type cleaner struct { + log *logp.Logger +} + +// run starts a loop that tries to clean entries from the registry. +// The cleaner locks the store, such that no new states can be created +// during the cleanup phase. Only resources that are finished and whos TTL +// (clean_timeout setting) has expired will be removed. +// +// Resources are considered "Finished" if they do not have a current owner (active input), and +// if they have no pending updates that still need to be written to the registry file after associated +// events have been ACKed by the outputs. +// The event acquisition timestamp is used as reference to clean resources. If a resources was blocked +// for a long time, and the life time has been exhausted, then the resource will be removed immediately +// once the last event has been ACKed. +func (c *cleaner) run(canceler unison.Canceler, store *store, interval time.Duration) { + panic("TODO: implement me") +} diff --git a/filebeat/input/v2/input-cursor/cursor.go b/filebeat/input/v2/input-cursor/cursor.go new file mode 100644 index 00000000000..2d30d386878 --- /dev/null +++ b/filebeat/input/v2/input-cursor/cursor.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +// Cursor allows the input to check if cursor status has been stored +// in the past and unpack the status into a custom structure. +type Cursor struct { + store *store + resource *resource +} + +func makeCursor(store *store, res *resource) Cursor { + return Cursor{store: store, resource: res} +} + +// IsNew returns true if no cursor information has been stored +// for the current Source. +func (c Cursor) IsNew() bool { return c.resource.IsNew() } + +// Unpack deserialized the cursor state into to. Unpack fails if no pointer is +// given, or if the structure to points to is not compatible with the document +// stored. +func (c Cursor) Unpack(to interface{}) error { + if c.IsNew() { + return nil + } + return c.resource.UnpackCursor(to) +} diff --git a/filebeat/input/v2/input-cursor/doc.go b/filebeat/input/v2/input-cursor/doc.go new file mode 100644 index 00000000000..1c6f399ce45 --- /dev/null +++ b/filebeat/input/v2/input-cursor/doc.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package cursor provides an InputManager for use with the v2 API, that is +// capable of storing an internal cursor state between restarts. +// +// The InputManager requires authors to Implement a configuration function and +// the cursor.Input interface. The configuration function returns a slice of +// sources ([]Source) that it has read from the configuration object, and the +// actual Input that will be used to collect events from each configured +// source. +// When Run a go-routine will be started per configured source. If two inputs have +// configured the same source, only one will be active, while the other waits +// for the resource to become free. +// The manager keeps track of the state per source. When publishing an event a +// new cursor value can be passed as well. Future instance of the input can +// read the last published cursor state. +// +// For each source an in-memory and a persitent state are tracked. Internal +// meta updates by the input manager can not be read by Inputs, and will be +// written to the persistent store immediately. Cursor state updates are read +// and update by the input. Cursor updates are written to the persistent store +// only after the events have been ACKed by the output. Internally the input +// manager keeps track of already ACKed updates and pending ACKs. +// In order to guarantee progress even if the pbulishing is slow or blocked, all cursor +// updates are written to the in-memory state immediately. Source without any +// pending updates are in-sync (in-memory state == persistet state). All +// updates are ordered, but we allow the in-memory state to be ahead of the +// persistent state. +// When an input is started, the cursor state is read from the in-memory state. +// This way a new input instance can continue where other inputs have been +// stopped, even if we still have in-flight events from older input instances. +// The coordination between inputs guarantees that all updates are always in +// order. +// +// When a shutdown signal is received, the publisher is directly disconnected +// from the outputs. As all coordination is directly handled by the +// InputManager, shutdown will be immediate (once the input itself has +// returned), and can not be blocked by the outputs. +// +// An input that is about to collect a source that is already collected by +// another input will wait until the other input has returned or the current +// input did receive a shutdown signal. +package cursor diff --git a/filebeat/input/v2/input-cursor/input.go b/filebeat/input/v2/input-cursor/input.go new file mode 100644 index 00000000000..94faeeaadda --- /dev/null +++ b/filebeat/input/v2/input-cursor/input.go @@ -0,0 +1,84 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "fmt" + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" +) + +// Input interface for cursor based inputs. This interface must be implemented +// by inputs that with to use the InputManager in order to implement a stateful +// input that can store state between restarts. +type Input interface { + Name() string + + // Test checks the configuaration and runs additional checks if the Input can + // actually collect data for the given configuration (e.g. check if host/port or files are + // accessible). + // The input manager will call Test per configured source. + Test(Source, input.TestContext) error + + // Run starts the data collection. Run must return an error only if the + // error is fatal making it impossible for the input to recover. + // The input run a go-routine can call Run per configured Source. + Run(input.Context, Source, Cursor, Publisher) error +} + +// managedInput implements the v2.Input interface, integrating cursor Inputs +// with the v2 input API. +// The managedInput starts go-routines per configured source. +// If a Run returns the error is 'remembered', but active data collecting +// continues. Only after all Run calls have returned will the managedInput be +// done. +type managedInput struct { + manager *InputManager + userID string + sources []Source + input Input + cleanTimeout time.Duration +} + +// Name is required to implement the v2.Input interface +func (inp *managedInput) Name() string { return inp.input.Name() } + +// Test runs the Test method for each configured source. +func (inp *managedInput) Test(ctx input.TestContext) error { + panic("TODO: implement me") +} + +// Run creates a go-routine per source, waiting until all go-routines have +// returned, either by error, or by shutdown signal. +// If an input panics, we create an error value with stack trace to report the +// issue, but not crash the whole process. +func (inp *managedInput) Run( + ctx input.Context, + pipeline beat.PipelineConnector, +) (err error) { + panic("TODO: implement me") +} + +func (inp *managedInput) createSourceID(s Source) string { + if inp.userID != "" { + return fmt.Sprintf("%v::%v::%v", inp.manager.Type, inp.userID, s.Name()) + } + return fmt.Sprintf("%v::%v", inp.manager.Type, s.Name()) +} diff --git a/filebeat/input/v2/input-cursor/manager.go b/filebeat/input/v2/input-cursor/manager.go new file mode 100644 index 00000000000..ee1b2bc7939 --- /dev/null +++ b/filebeat/input/v2/input-cursor/manager.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + + "github.com/elastic/go-concert/unison" +) + +// InputManager is used to create, manage, and coordinate stateful inputs and +// their persistent state. +// The InputManager ensures that only one input can be active for a unique source. +// If two inputs have overlapping sources, both can still collect data, but +// only one input will collect from the common source. +// +// The InputManager automatically cleans up old entries without an active +// input, and without any pending update operations for the persistent store. +// +// The Type field is used to create the key name in the persistent store. Users +// are allowed to add a custome per input configuration ID using the `id` +// setting, to collect the same source multiple times, but with different +// state. The key name in the persistent store becomes -[]- +type InputManager struct { + Logger *logp.Logger + + // StateStore gives the InputManager access to the persitent key value store. + StateStore StateStore + + // Type must contain the name of the input type. It is used to create the key name + // for all sources the inputs collect from. + Type string + + // DefaultCleanTimeout configures the key/value garbage collection interval. + // The InputManager will only collect keys for the configured 'Type' + DefaultCleanTimeout time.Duration + + // Configure returns an array of Sources, and a configured Input instances + // that will be used to collect events from each source. + Configure func(cfg *common.Config) ([]Source, Input, error) + + store *store +} + +// Source describe a source the input can collect data from. +// The `Name` method must return an unique name, that will be used to identify +// the source in the persistent state store. +type Source interface { + Name() string +} + +// StateStore interface and configurations used to give the Manager access to the persistent store. +type StateStore interface { + Access() (*statestore.Store, error) + CleanupInterval() time.Duration +} + +// Init starts background processes for deleting old entries from the +// persistent store if mode is ModeRun. +func (cim *InputManager) Init(group unison.Group, mode v2.Mode) error { + panic("TODO: implement me") +} + +// Create builds a new v2.Input using the provided Configure function. +// The Input will run a go-routine per source that has been configured. +func (cim *InputManager) Create(config *common.Config) (input.Input, error) { + panic("TODO: implement me") +} + +func lockResource(log *logp.Logger, resource *resource, canceler input.Canceler) error { + if !resource.lock.TryLock() { + log.Infof("Resource '%v' currently in use, waiting...", resource.key) + err := resource.lock.LockContext(canceler) + if err != nil { + log.Infof("Input for resource '%v' has been stopped while waiting", resource.key) + return err + } + } + return nil +} + +func releaseResource(resource *resource) { + resource.lock.Unlock() + resource.Release() +} diff --git a/filebeat/input/v2/input-cursor/publish.go b/filebeat/input/v2/input-cursor/publish.go new file mode 100644 index 00000000000..437c4e07373 --- /dev/null +++ b/filebeat/input/v2/input-cursor/publish.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" +) + +// Publisher is used to publish an event and update the cursor in a single call to Publish. +// Inputs are allowed to pass `nil` as cursor state. In this case the state is not updated, but the +// event will still be published as is. +type Publisher interface { + Publish(event beat.Event, cursor interface{}) error +} + +// cursorPublisher implements the Publisher interface and used internally by the managedInput. +// When publishing an event with cursor state updates, the cursorPublisher +// updates the in memory state and create an updateOp that is used to schedule +// an update for the persistent store. The updateOp is run by the inputs ACK +// handler, persisting the pending update. +type cursorPublisher struct { + canceler input.Canceler + client beat.Client + cursor *Cursor +} + +// updateOp keeps track of pending updates that are not written to the persistent store yet. +// Update operations are ordered. The input manager guarantees that only one +// input can create update operation for a source, such that new input +// instances can add update operations to be executed after already pending +// update operations from older inputs instances that have been shutdown. +type updateOp struct { + store *store + resource *resource + + // state updates to persist + timestamp time.Time + ttl time.Duration + delta interface{} +} + +// Publish publishes an event. Publish returns false if the inputs cancellation context has been marked as done. +// If cursorUpdate is not nil, Publish updates the in memory state and create and updateOp for the pending update. +// It overwrite event.Private with the update operation, before finally sending the event. +// The ACK ordering in the publisher pipeline guarantees that update operations +// will be ACKed and executed in the correct order. +func (c *cursorPublisher) Publish(event beat.Event, cursorUpdate interface{}) error { + panic("TODO: implement me") +} + +// Execute updates the persistent store with the scheduled changes and releases the resource. +func (op *updateOp) Execute(numEvents uint) { + panic("TODO: implement me") +} diff --git a/filebeat/input/v2/input-cursor/store.go b/filebeat/input/v2/input-cursor/store.go new file mode 100644 index 00000000000..f44cf6760dc --- /dev/null +++ b/filebeat/input/v2/input-cursor/store.go @@ -0,0 +1,203 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "sync" + "time" + + "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/go-concert" + "github.com/elastic/go-concert/unison" +) + +// store encapsulates the persistent store and the in memory state store, that +// can be ahead of the the persistent store. +// The store lifetime is managed by a reference counter. Once all owners (the +// session, and the resource cleaner) have dropped ownership, backing resources +// will be released and closed. +type store struct { + log *logp.Logger + refCount concert.RefCount + persistentStore *statestore.Store + ephemeralStore *states +} + +// states stores resource states in memory. When a cursor for an input is updated, +// it's state is updated first. The entry in the persistent store 'follows' the internal state. +// As long as a resources stored in states is not 'Finished', the in memory +// store is assumed to be ahead (in memory and persistent state are out of +// sync). +type states struct { + mu sync.Mutex + table map[string]*resource +} + +// resource holds the in memory state and keeps track of pending updates and inputs collecting +// event for the resource its key. +// A resource is assumed active for as long as at least one input has (or tries +// to) acuired the lock, and as long as there are pending updateOp instances in +// the pipeline not ACKed yet. The key can not gc'ed by the cleaner, as long as the resource is active. +// +// State chagnes and writes to the persistent store are protected using the +// stateMutex, to ensure full consistency between direct writes and updates +// after ACK. +type resource struct { + // pending counts the number of Inputs and outstanding registry updates. + // as long as pending is > 0 the resource is in used and must not be garbage collected. + pending atomic.Uint64 + + // lock guarantees only one input create updates for this entry + lock unison.Mutex + + // key of the resource as used for the registry. + key string + + // stateMutex is used to lock the resource when it is update/read from + // multiple go-routines like the ACK handler or the input publishing an + // event. + // stateMutex is used to access the fields 'stored', 'state' and 'internalInSync' + stateMutex sync.Mutex + + // stored indicates that the state is available in the registry file. It is false for new entries. + stored bool + + // internalInSync is true if all 'Internal' metadata like TTL or update timestamp are in sync. + // Normally resources are added when being created. But if operations failed we will retry inserting + // them on each update operation until we eventually succeeded + internalInSync bool + + activeCursorOperations uint + internalState stateInternal + + // cursor states. The cursor holds the state as it is currently known to the + // persistent store, while pendingCursor contains the most recent update + // (in-memory state), that still needs to be synced to the persistent store. + // The pendingCursor is nil if there are no pending updates. + // When processing update operations on ACKs, the state is applied to cursor + // first, which is finally written to the persistent store. This ensures that + // we always write the complete state of the key/value pair. + cursor interface{} + pendingCursor interface{} +} + +type ( + // state represents the full document as it is stored in the registry. + // + // The TTL and Update fields are for internal use only. + // + // The `Cursor` namespace is used to store the cursor information that are + // required to continue processing from the last known position. Cursor + // updates in the registry file are only executed after events have been + // ACKed by the outputs. Therefore the cursor MUST NOT include any + // information that are require to identify/track the source we are + // collecting from. + state struct { + TTL time.Duration + Updated time.Time + Cursor interface{} + } + + stateInternal struct { + TTL time.Duration + Updated time.Time + } +) + +// hook into store close for testing purposes +var closeStore = (*store).close + +func openStore(log *logp.Logger, statestore StateStore, prefix string) (*store, error) { + panic("TODO: implement me") +} + +func (s *store) Retain() { s.refCount.Retain() } +func (s *store) Release() { + if s.refCount.Release() { + closeStore(s) + } +} + +func (s *store) close() { + panic("TODO: implement me") +} + +// Get returns the resource for the key. +// A new shared resource is generated if the key is not known. The generated +// resource is not synced to disk yet. +func (s *store) Get(key string) *resource { + return s.ephemeralStore.Find(key, true) +} + +// UpdateTTL updates the time-to-live of a resource. Inactive resources with expired TTL are subject to removal. +// The TTL value is part of the internal state, and will be written immediately to the persistent store. +// On update the resource its `cursor` state is used, to keep the cursor state in sync with the current known +// on disk store state. +func (s *store) UpdateTTL(resource *resource, ttl time.Duration) { + panic("TODO: implement me") +} + +// Find returns the resource for a given key. If the key is unknown and create is set to false nil will be returned. +// The resource returned by Find is marked as active. (*resource).Release must be called to mark the resource as inactive again. +func (s *states) Find(key string, create bool) *resource { + panic("TODO: implement me") +} + +// IsNew returns true if we have no state recorded for the current resource. +func (r *resource) IsNew() bool { + panic("TODO: implement me") +} + +// Retain is used to indicate that 'resource' gets an additional 'owner'. +// Owners of an resource can be active inputs or pending update operations +// not yet written to disk. +func (r *resource) Retain() { r.pending.Inc() } + +// Release reduced the owner ship counter of the resource. +func (r *resource) Release() { r.pending.Dec() } + +// UpdatesReleaseN is used to release ownership of N pending update operations. +func (r *resource) UpdatesReleaseN(n uint) { + r.pending.Sub(uint64(n)) +} + +// Finished returns true if the resource is not in use and if there are no pending updates +// that still need to be written to the registry. +func (r *resource) Finished() bool { return r.pending.Load() == 0 } + +// UnpackCursor deserializes the in memory state. +func (r *resource) UnpackCursor(to interface{}) error { + r.stateMutex.Lock() + defer r.stateMutex.Unlock() + if r.activeCursorOperations == 0 { + return typeconv.Convert(to, r.cursor) + } + return typeconv.Convert(to, r.pendingCursor) +} + +// syncStateSnapshot returns the current insync state based on already ACKed update operations. +func (r *resource) inSyncStateSnapshot() state { + return state{ + TTL: r.internalState.TTL, + Updated: r.internalState.Updated, + Cursor: r.cursor, + } +}