From 00ed2d637f21522678232a8cf1f8e9c7dc97ae40 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Thu, 10 Aug 2023 11:37:56 -0300 Subject: [PATCH] docs(filters): added legacy --- docs/pages/v2/filters.mdx | 5 +- docs/pages/v2/filters/fingerprint.mdx | 22 --- docs/pages/v2/filters/legacy_v1.mdx | 61 ++++++ docs/pages/v2/filters/selection.mdx | 274 -------------------------- 4 files changed, 62 insertions(+), 300 deletions(-) delete mode 100644 docs/pages/v2/filters/fingerprint.mdx create mode 100644 docs/pages/v2/filters/legacy_v1.mdx delete mode 100644 docs/pages/v2/filters/selection.mdx diff --git a/docs/pages/v2/filters.mdx b/docs/pages/v2/filters.mdx index 8e25c5c0..bec90d3b 100644 --- a/docs/pages/v2/filters.mdx +++ b/docs/pages/v2/filters.mdx @@ -9,9 +9,6 @@ These are the existing filters that are included as part the main _Oura_ codebas - [ParseCbor](filters/parse_cbor.mdx): a filter that maps cbor transactions to a data structure. - [SplitBlock](filters/split_block.mdx): a filter that will decode the cbor block and extract all transactions in an event in the format CborTx. - [Deno](filters/deno.mdx): a filter that allows JS code to be implemented as a stage within the pipeline. - -### Legacy V1 -- [Fingerprint](filters/fingerprint.mdx): a filter that adds a (probably) unique identifier (fingerprint) to each event. -- [Selection](filters/selection.mdx): a filter that evaluates different built-in predicates to decide which events should go through the pipeline. +- [Legacy V1](filters/legacy_v1.mdx): a filter that transforms the block data to the Oura V1 data structure. New filters are being developed, information will be added in this documentation to reflect the updated list. Contributions and feature request are welcome in our [Github Repo](https://github.com/txpipe/oura). diff --git a/docs/pages/v2/filters/fingerprint.mdx b/docs/pages/v2/filters/fingerprint.mdx deleted file mode 100644 index d872c445..00000000 --- a/docs/pages/v2/filters/fingerprint.mdx +++ /dev/null @@ -1,22 +0,0 @@ -# Fingerprint Filter - -A filter that computes a (probably) unique identifier for each event and appends it as part of the data passed forward. - -Dealing with duplicated records is a common problem in data pipelines. A common workaround is to use identifiers based on the hash of the data for each record. In this way, a duplicated record would yield the same hash, allowing the storage engine to discard the extra instance. - -The _fingerprint_ filter uses the non-cryptographic hash algorithm `murmur3` to compute an id for each _Oura_ event with a very low collision level. We use a non-cryptographic hash because they are faster to compute and non of the cryptographic properties are required for this use-case. - -When enabled, this filter will set the `fingerprint` property of the `Event` data structure passed through each stage of the pipeline. _Sinks_ at the end of the process might leverage this value as primary key of the corresponding storage mechanism. - -## Configuration - -Adding the following section to the daemon config file will enable the filter as part of the pipeline: - -```toml -[[filters]] -type = "Fingerprint" -``` - -### Section: `filter` - -- `type`: the literal value `Fingerprint`. diff --git a/docs/pages/v2/filters/legacy_v1.mdx b/docs/pages/v2/filters/legacy_v1.mdx new file mode 100644 index 00000000..6b55a41b --- /dev/null +++ b/docs/pages/v2/filters/legacy_v1.mdx @@ -0,0 +1,61 @@ +# Legacy V1 filter + +Transforms the block data to the Oura V1 data structure. This filter is ideal for compatibility with systems that are using Oura V1 and want to migrate without much work. + +## Configuration + +Adding the following section to the daemon config file will enable the filter as part of the pipeline: + +```toml +[[filters]] +type = "LegacyV1" +include_block_end_events = false +include_transaction_details = false +include_transaction_end_events = false +include_block_details = false +include_block_cbor = false +include_byron_ebb = false +``` + +### Section `LegacyV1` + +- `type`: the literal value `LegacyV1`. +- `include_block_end_events`: if enabled, the source will output an event signaling the end of a block, duplicating all of the data already sent in the corresponding block start event. Default value is `false`. +- `include_transaction_details`: if enabled, each transaction event payload will contain an nested version of all of the details of the transaction (inputs, outputs, mint, assets, metadata, etc). Useful when the pipeline needs to process the tx as a unit, instead of handling each sub-object as an independent event. Default value is `false`. +- `include_transaction_end_events`: if enabled, the source will output an event signaling the end of a transaction, duplicating all of the data already sent in the corresponding transaction start event. Defaul value is `false`. +- `include_block_cbor`: if enabled, the block event will include the raw, unaltered cbor content received from the node, formatted as an hex string. Useful when some custom cbor decoding is required. Default value is `false`. +- `include_byron_ebb`: if enabled, a block event will be emmitted for legacy epoch boundary block of the Byron era (deprecated in newer eras). Useful when performing validation on previous block hashes. Default value is `false`. +- `include_block_details`: If enabled, will be added the basic details of each transaction. Default value is `false`. + +## Examples + +Below is an example of the data that will be sent to the sink with all settings disabled. + +```json +{ + "event": "apply", + "point": { + "slot": 100110525, + "hash": "c808fc4142c5f10a2a6d0922edbd23972100d7d22e2255206bd05e968cc045f1" + }, + "record": { + "context": { + "block_hash": "c808fc4142c5f10a2a6d0922edbd23972100d7d22e2255206bd05e968cc045f1", + "block_number": 9142145, + "slot": 100110525, + "timestamp": 1691676816, + "tx_idx": 6, + "tx_hash": "4329140c6711f2197c8c81bfff4b75fb95892375050dafda30ba146476ca3d65", + "input_idx": null, + "output_idx": null, + "output_address": null, + "certificate_idx": null + }, + "collateral": { + "tx_id": "981881a382a41f85140076b088a93456a999e6cd2796a28c0a8c7858cea40175", + "index": 4 + }, + "fingerprint": null + } +} +``` diff --git a/docs/pages/v2/filters/selection.mdx b/docs/pages/v2/filters/selection.mdx deleted file mode 100644 index d7359e81..00000000 --- a/docs/pages/v2/filters/selection.mdx +++ /dev/null @@ -1,274 +0,0 @@ -### Table of contents -- [Selection filter](#selection-filter) -- [General predicates](#general-predicates) -- [Variant-restricted predicates](#variant-restricted-predicates) -- [Real world example](#real-world-example) -# Selection filter - -A filter that evaluates a set of configurable predicates against each event in the pipeline to decide which records should be sent to the following stage. - -Not every use-case requires each and every event to be processed. For example, a pipeline interested in creating a 'metadata' search engine might not care about transaction outputs. With a similar logic, a pipeline aggregating transaction amounts might not care about metadata. The _selection filter_ provides a way to optimize the pipeline so that only relevant events are processed. - -The filter works by evaluating a predicate against each event. If the predicate returns `true`, then the event will continue down the pipeline. If the predicate evalutes to `false`, the event will be dopped. We currently provide some common built-in predicate to facilitate common use-cases (eg: matching event type, matching policy id, matching a metadata key, etc.). We also provide some 'connecting' predicates like `all_of`, `any_of`, and `not` which can be used to create complex conditions by composing other predicates. - -## Configuration - -Adding the following section to the daemon config file will enable the filter as part of the pipeline: - -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "" -argument = -``` - -### Section: `filters` - -- `type`: the literal value `Selection`. - -### Section `filters.check` - -- `predicate`: the key of the predicate to use for the evaluation. See the list of available predicates for possible values. -- `argument`: a polimorphic argument that specializes the behavior of the predicate in some way. - -## General predicates -Predicates available for events of any type. - -### `variant_in (string[])` - This predicate will yield true when the variant of the event matches any of the items in the argument array. Available variants: -- [Block](../reference/data_dictionary.md#block-event) -- [CIP15Asset](../reference/data_dictionary.md#cip15asset-event) -- [CIP25Asset](../reference/data_dictionary.md#cip25asset-event) -- [Collateral](../reference/data_dictionary.md#collateral-event) -- [GenesisKeyDelegation](../reference/data_dictionary.md#genesiskeydelegation-event) -- [Metadata](../reference/data_dictionary.md#metadata-event) -- [Mint](../reference/data_dictionary.md#mint-event) -- [MoveInstantaneousRewardsCert](../reference/data_dictionary.md#moveinstantaneousrewardscert-event) -- [NativeScript](../reference/data_dictionary.md#nativescript-event) -- [NativeWitness](../reference/data_dictionary.md#nativewitness-event) -- [OutputAsset](../reference/data_dictionary.md#outputasset-event) -- [PlutusDatum](../reference/data_dictionary.md#plutusdatum-event) -- [PlutusRedeemer](../reference/data_dictionary.md#plutusredeemer-event) -- [PlutusScript](../reference/data_dictionary.md#plutusscript-event) -- [PlutusWitness](../reference/data_dictionary.md#plutuswitness-event) -- [PoolRegistration](../reference/data_dictionary.md#poolregistration-event) -- [PoolRetirement](../reference/data_dictionary.md#poolretirement-event) -- [RollBack](../reference/data_dictionary.md#rollback-event) -- [StakeDelegation](../reference/data_dictionary.md#stakedelegation-event) -- [StakeDeregistration](../reference/data_dictionary.md#stakederegistration-event) -- [StakeRegistration](../reference/data_dictionary.md#stakeregistration-event) -- [Transaction](../reference/data_dictionary.md#transaction-event) -- [TxInput](../reference/data_dictionary.md#txinput-event) -- [TxOutput](../reference/data_dictionary.md#txoutput-event) -- [VKeyWitness](../reference/data_dictionary.md#vkeywitness-event) - -**Example** - Allowing only block and transaction events to pass: - -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "variant_in" -argument = ["Block", "Transaction"] -``` - -### `variant_not_in (string[])` - This predicate will yield true when the variant of the event doesn't match any of the items in the argument array. - -**Example** - Allowing all events except transaction to pass: -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "variant_not_in" -argument = ["Transaction"] -``` - -### `not (predicate)` - This predicate will yield true when the predicate in the arguments yields false. - - -**Example** - Using the `not` predicate to allow all events except the variant `Transaction`: - -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "not" - -[filters.check.argument] -predicate = "variant_in" -argument = ["Transaction"] -``` - -### `any_of (predicate[])` - This predicate will yield true when _any_ of the predicates in the argument yields true. - - -**Example** - Using the `any_of` predicate to filter events presenting any of two different policies (Boolean "or"): - -```toml -[filters.check] -predicate = "any_of" - -[[filters.check.argument]] -predicate = "policy_equals" -argument = "4bf184e01e0f163296ab253edd60774e2d34367d0e7b6cbc689b567d" - -[[filters.check.argument]] -predicate = "policy_equals" -argument = "a5bb0e5bb275a573d744a021f9b3bff73595468e002755b447e01559" -``` - -### `all_of (predicate[])` - This predicate will yield true when _all_ of the predicates in the argument yields true. - -**Example** - Using the `all_of` predicate to filter only "asset" events presenting a particular policy (Boolean "and") : - -```toml -[filters.check] -predicate = "all_of" - -[[filters.check.argument]] -predicate = "variant_in" -argument = ["OutputAsset"] - -[[filters.check.argument]] -predicate = "policy_equals" -argument = "a5bb0e5bb275a573d744a021f9b3bff73595468e002755b447e01559" -``` - -## Variant-restricted predicates -Predicates operating on a subset of event variants. - - -### `policy_equals (string)` - This predicate will yield true when the policy of a mint or output asset matches the value in the argument. - - -**Variants:** `Transaction`, `Mint`, `CIP25Asset`, `OutputAsset` - -**Example** -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "policy_equals" -argument = "" -``` - -### `asset_equals (string)` - This predicate will yield true when the asset (token name) of a mint or output asset matches the value in the argument. - -**Variants:** `CIP25Asset`, `Transaction`, `OutputAsset`, `Mint` - -**Example** -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "asset_equals" -argument = "" -``` - -### `metadata_label_equals (string)` - This predicate will yield true when the root label of a metadata entry matches the value in the argument. - -**Variants:** `Metadata`, `Transaction` - -**Example** -```toml -[[filters]] -type = "Selection" - -[filters.check] -predicate = "metadata_label_equals" -argument = "