From 4673368d1d7938311b59633c0a97084084cb26c4 Mon Sep 17 00:00:00 2001 From: Brandon Ramirez Date: Mon, 26 Mar 2018 20:47:53 -0700 Subject: [PATCH 1/3] Add example mapping for old OriginListing entity using event sourced approach --- specs/ingest-mapping/examples/IPFS.yaml | 50 +++++++++++++++++ .../EventSource-GenericDSL.yaml | 55 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 specs/ingest-mapping/examples/IPFS.yaml create mode 100644 specs/ingest-mapping/examples/OriginListing_old/EventSource-GenericDSL.yaml diff --git a/specs/ingest-mapping/examples/IPFS.yaml b/specs/ingest-mapping/examples/IPFS.yaml new file mode 100644 index 0000000..6f5b45b --- /dev/null +++ b/specs/ingest-mapping/examples/IPFS.yaml @@ -0,0 +1,50 @@ +# Contract source: https://github.com/OriginProtocol/demo-dapp/blob/master/contracts/Listing.sol +# specVersion indicates the version of the ingest mapping spec +specVersion: 0.0.1 +# definitions are optional to help readability. May be used to specify aliases +# for use later in the file. YAML requires that aliases be declared before they +# are referenced. +definitions: +# Links to the GraphQL schema being mapped to +# i.e. +# type User { +# id: ID +# firstName: String +# lastName: String +# } +schema: + /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k +datasets: + - data: ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k + structure: + format: text/csv + # https://frictionlessdata.io/specs/tabular-data-package/ + schema: + fields: + - name: id + type: integer + - name: first_name + type: string + - name: last_name + type: string + - name: age + type: integer + charEncoding: utf-8 + mapping: + # Strategy specifies how we will access/decode/parse the storage. This + # would be the extension point for implementing new "strategies" for + # parsing storage with custom logic. In the future, there might be a + # "Javascript" strategy where the user interacts with the contract through + # a web3 API + - strategy: state-mapping + # Defines transofrmations on the input data to fit the final schema + transformState: + mapKeys: + - from: first_name + to: firstName + - from: last_name + to: lastName + pluck: + - id + - firstName + - lastName diff --git a/specs/ingest-mapping/examples/OriginListing_old/EventSource-GenericDSL.yaml b/specs/ingest-mapping/examples/OriginListing_old/EventSource-GenericDSL.yaml new file mode 100644 index 0000000..42b51d9 --- /dev/null +++ b/specs/ingest-mapping/examples/OriginListing_old/EventSource-GenericDSL.yaml @@ -0,0 +1,55 @@ +# Contract source: https://github.com/OriginProtocol/demo-dapp/blob/master/contracts/Listing.sol +# specVersion indicates the version of the ingest mapping spec +specVersion: 0.0.1 +# definitions are optional to help readability. May be used to specify aliases +# for use later in the file. YAML requires that aliases be declared before they +# are referenced. +definitions: +# Links to the GraphQL schema being mapped to +schema: + # An IPLD link. This may not be needed if we have a single global schema. + /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k +# A single DataSource may comprise multiple DataSets. +# DataSets are composed of Data, Structure & Mapping, which are intentionally +# decoupled. +datasets: + # IPLD address of the data for the dataset + - data: eth-contract/0x06012c8cf97BEaD5deAe237070F9587f8E7A266d + structure: + format: eth-contract + abi: + # This is an IPLD link. Any value can be replaced with an IPLD link for + # improved legibility. See: https://github.com/ipld/specs/tree/master/ipld + /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k + mapping: + strategy: eth-contract-event-source + # How to access entities using smart contract interface. + # Not the source of truth on current state, but used in event source logic. + accessors: + - entity: "@OriginProtocol/Listing" + # getListing is a public method on the OriginProtocol Listing contract + # for accessing the Listing entity + getByIndex: getListing + # This is where the core of the event sourcing logic happens using pipeline + # operators. The events being sourced are in the ethereum contract event log. + processEvents: + - fork: + - filter: + event: NewListing + - getEntityByIndex: + entity: "@OriginProtocol/Listing" + idField: id + - createEntity: + entity: "@OriginProtocol/Listing" + idField: id + - fork: + - filter: + event: ListingPurchased + # Gets the state *after* the Listing was updated. + - getEntityByIndex: + entity: "@OriginProtocol/Listing" + index: _index + # Updates an entity in The Graph by ID + - updateEntity: + entity: "@OriginProtocol/Listing" + idField: id From ee1ca9037c4ef989dcbd638fd4d1ec5ca4adf4f7 Mon Sep 17 00:00:00 2001 From: Brandon Ramirez Date: Tue, 10 Apr 2018 10:32:39 -0700 Subject: [PATCH 2/3] Add example old OriginProtocol mapping using specialized DSL --- .../StateSnapshot-SpecializedDSL.yaml | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 specs/ingest-mapping/examples/OriginListing_old/StateSnapshot-SpecializedDSL.yaml diff --git a/specs/ingest-mapping/examples/OriginListing_old/StateSnapshot-SpecializedDSL.yaml b/specs/ingest-mapping/examples/OriginListing_old/StateSnapshot-SpecializedDSL.yaml new file mode 100644 index 0000000..cf40dba --- /dev/null +++ b/specs/ingest-mapping/examples/OriginListing_old/StateSnapshot-SpecializedDSL.yaml @@ -0,0 +1,153 @@ +specVersion: mapping/v1 +mappings: + - kind: ethereum/array-collection + entity: ForSaleListing + + # Collection name and size define the state variables or + # functions that provide the array collection and its length + collectionName: listings + collectionSize: listingsLength + + # Fields defines fields that can be retrieved from the + # structs returned for each item in the array collection; + # + # Fields starting with an underscore (e.g. _ipfsHash) are + # considered internal and are stripped of before passing + # entities on to the indexer's data source event handler + fields: + lister: + kind: ethereum/struct-field-by-index + index: 0 + _ipfsHash: + kind: ethereum/struct-field-by-index + index: 1 + price: + kind: ethereum/struct-field-by-index + index: 2 + unitsAvailable: + kind: ethereum/struct-field-by-index + index: 3 + + # Extra fields are additional fields that can come from additional + # smart contract function calls, external files e.g. on IPFS and more; + # they can refer to values of regular fields + extraFields: + id: + kind: ethereum/array-collection-item-index + _schema: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "schema"] + category: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "category"] + description: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "description"] + + # Transformations are applied to entities after all of their + # fields have been collected; in this example, these items + # are filtered based on a filter called is-for-sale-listing, which + # is defined further below + transformations: + - kind: filter + name: is-for-sale-listing + + # Events describe events that can lead to entities being reindexed + events: + - name: NewListing + pipeline: + - kind: ethereum/event-param + name: _index + - kind: ethereum/array-collection-item + - kind: filter + name: is-for-sale-listing + - kind: update-entity + - name: ListingPurchased + pipeline: + - kind: ethereum/event-param + name: _index + - kind: ethereum/array-collection-item + - kind: filter + name: is-for-sale-listing + - kind: update-entity + + # Filters can be used in transformations and events and are defined + # in a single place + filters: + is-for-sale-listing: + kind: filter-by-field-value + field: _schema + value: https://localhost:3000/schemas/for-sale.json + + - kind: ethereum/array-collection + entity: AnnouncementListing + collectionName: listings + collectionSize: listingsLength + fields: + lister: + kind: ethereum/struct-field-by-index + index: 0 + _ipfsHash: + kind: ethereum/struct-field-by-index + index: 1 + price: + kind: ethereum/struct-field-by-index + index: 2 + unitsAvailable: + kind: ethereum/struct-field-by-index + index: 3 + extraFields: + id: + kind: ethereum/array-collection-item-index + _schema: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "schema"] + category: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "category"] + description: + kind: ipfs/field-in-json-file + hash: + source: field + name: _ipfsHash + path: ["data", "description"] + transformations: + - kind: filter + name: filter-announcement + events: + - name: NewListing + pipeline: + - kind: ethereum/event-param + name: _index + - kind: ethereum/array-collection-item + - kind: filter + name: is-announcement-listing + - kind: update-entity + - name: ListingPurchased + pipeline: + - kind: ethereum/event-param + name: _index + - kind: ethereum/array-collection-item + - kind: filter + name: is-announcement-listing + - kind: update-entity + filters: + is-announcement-listing: + kind: filter-by-field-value + field: _schema + value: https://localhost:3000/schemas/announcement.json From a6edb7a9e4b911e94c362a6af9d145b67deed51e Mon Sep 17 00:00:00 2001 From: Brandon Ramirez Date: Wed, 11 Apr 2018 16:47:28 -0700 Subject: [PATCH 3/3] Add example mapping for old OriginListing contract in Javascript --- .../examples/OriginListing_old/ETL.js | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 specs/ingest-mapping/examples/OriginListing_old/ETL.js diff --git a/specs/ingest-mapping/examples/OriginListing_old/ETL.js b/specs/ingest-mapping/examples/OriginListing_old/ETL.js new file mode 100644 index 0000000..26c4102 --- /dev/null +++ b/specs/ingest-mapping/examples/OriginListing_old/ETL.js @@ -0,0 +1,114 @@ +import { map, range } from "lodash/fp"; + +/** + * Extracts a list of objects representing one or more entity types from a storage backend. + * Is injected with clients for the various available storage backends. + * Does not use actual Ethereum or IPFS addresses. + * @param {[type]} web3 A client for interacting with an Ethereum storage backend. + * @param {[type]} ipfs A client for interacting with an IPFS storage backend. + */ +export const extract = ({ web3, ipfs }) => { + const abiCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ"; + return ipfs.block + .get(abiCid) + .then(block => { + const abi = JSON.parse(block.data); + const OriginListing = web3.eth.contract(abi); + const originListing = OriginListing.at( + "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" + ); + return originListing.listingsLength(); + }) + .then(range(0)) + .then(map(value => contract.getListing(value))) + .then( + map(([index, lister, ipfsHash, price, unitsAvailable]) => ({ + index, + lister, + ipfsHash, + price, + unitsAvailable + })) + ) + .then(entities => ( + Promise.all( + entities.map(entity => ( + // OriginListing contract uses truncated IPFS hashes + ipfs.block.get(`Qm${entity.ipfsHash}`) + .then(data => JSON.parse(data)); + .then(ipfsObject => ({ + index: entity.index, + lister: entity.lister, + price: entity.price, + unitsAvailable: unitsAvailable, + listingData: ipfsObject.data, + })) + )) + ); + )); +}; + +/** + * A transform function to be applied to each entity individually that is returned + * from the extract function. + * @param {[type]} entity [description] + */ +export const transform = (entity) => { + const {index, lister, price, unitsAvailable, listingData} = entity + return { + ...listingData, + id: index, + lister, + price, + unitsAvailable, + } +}; + +/** + * A load function which loads entities of a specific type. Final step in the + * ETL pipeline. + * + * @param {[type]} entity An entity returned from the transform stage of ETL pipeline. + * @param {[type]} loadEntity A function to load entities, accepts entityType and entityData + */ +export const load = (entity, loadEntity) => { + switch(entity.category) { + case "ForSale": + createEntity('ForSaleListing', entity) + case "Housing": + createEntity('HousingListing', entity) + default: + } +}; + +/** + * The update function defines when reindexing should be triggered based on asynchronous + * events at the storage layer. + * @param {[type]} web3 A client for interacting with an Ethereum storage backend. + * @param {[type]} ipfs A client for interacting with an IPFS storage backend. + * @param {[type]} triggerUpdate A function to call anytime reindexing should occur. + */ +export const update = ({ ipfs, web3 }, triggerUpdate) => { + const abiCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ"; + return ipfs.block + .get(abiCid) + .then(block => { + const abi = JSON.parse(block.data); + const OriginListing = web3.eth.contract(abi); + const originListing = OriginListing.at( + "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" + ); + const newListing = originListing.NewListing(); + const listingPurchases = originListing.ListingPurchased(); + newListing.watch((err) => { + if (!err) { + triggerUpdate() + } + }) + listingUpdate.watch((err) => { + if (!err) { + triggerUpdate() + } + }) + }) +}