Merge pull request #2 from graphprotocol/zerim/spec/mappings

Add examples of ingest mapping for old OriginListing contract

specs/ingest-mapping/examples/IPFS.yaml

@@ -0,0 +1,50 @@
+# Contract source: https://github.com/OriginProtocol/demo-dapp/blob/master/contracts/Listing.sol
+# specVersion indicates the version of the ingest mapping spec
+specVersion: 0.0.1
+# definitions are optional to help readability. May be used to specify aliases
+# for use later in the file. YAML requires that aliases be declared before they
+# are referenced.
+definitions:
+# Links to the GraphQL schema being mapped to
+# i.e.
+# type User {
+#   id: ID
+#   firstName: String
+#   lastName: String
+# }
+schema:
+  /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k
+datasets:
+  - data: ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k
+    structure:
+      format: text/csv
+      # https://frictionlessdata.io/specs/tabular-data-package/
+      schema:
+        fields:
+          - name: id
+            type: integer
+          - name: first_name
+            type: string
+          - name: last_name
+            type: string
+          - name: age
+            type: integer
+      charEncoding: utf-8
+    mapping:
+        # Strategy specifies how we will access/decode/parse the storage. This
+        # would be the extension point for implementing new "strategies" for
+        # parsing storage with custom logic. In the future, there might be a
+        # "Javascript" strategy where the user interacts with the contract through
+        # a web3 API
+      - strategy: state-mapping
+        # Defines transofrmations on the input data to fit the final schema
+        transformState:
+          mapKeys:
+            - from: first_name
+              to: firstName
+            - from: last_name
+              to: lastName
+          pluck:
+            - id
+            - firstName
+            - lastName

specs/ingest-mapping/examples/OriginListing_old/ETL.js

@@ -0,0 +1,114 @@
+import { map, range } from "lodash/fp";
+
+/**
+ * Extracts a list of objects representing one or more entity types from a storage backend.
+ * Is injected with clients for the various available storage backends.
+ * Does not use actual Ethereum or IPFS addresses.
+ * @param  {[type]} web3 A client for interacting with an Ethereum storage backend.
+ * @param  {[type]} ipfs A client for interacting with an IPFS storage backend.
+ */
+export const extract = ({ web3, ipfs }) => {
+  const abiCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ";
+  return ipfs.block
+    .get(abiCid)
+    .then(block => {
+      const abi = JSON.parse(block.data);
+      const OriginListing = web3.eth.contract(abi);
+      const originListing = OriginListing.at(
+        "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d"
+      );
+      return originListing.listingsLength();
+    })
+    .then(range(0))
+    .then(map(value => contract.getListing(value)))
+    .then(
+      map(([index, lister, ipfsHash, price, unitsAvailable]) => ({
+        index,
+        lister,
+        ipfsHash,
+        price,
+        unitsAvailable
+      }))
+    )
+    .then(entities => (
+      Promise.all(
+        entities.map(entity => (
+          // OriginListing contract uses truncated IPFS hashes
+          ipfs.block.get(`Qm${entity.ipfsHash}`)
+          .then(data => JSON.parse(data));
+          .then(ipfsObject => ({
+            index: entity.index,
+            lister: entity.lister,
+            price: entity.price,
+            unitsAvailable: unitsAvailable,
+            listingData: ipfsObject.data,
+          }))
+        ))
+      );
+    ));
+};
+
+/**
+ * A transform function to be applied to each entity individually that is returned
+ * from the extract function.
+ * @param  {[type]} entity [description]
+ */
+export const transform = (entity) => {
+  const {index, lister, price, unitsAvailable, listingData} = entity
+  return {
+    ...listingData,
+    id: index,
+    lister,
+    price,
+    unitsAvailable,
+  }
+};
+
+/**
+ * A load function which loads entities of a specific type. Final step in the
+ * ETL pipeline.
+ *
+ * @param  {[type]} entity       An entity returned from the transform stage of ETL pipeline.
+ * @param  {[type]} loadEntity   A function to load entities, accepts entityType and entityData
+ */
+export const load = (entity, loadEntity) => {
+  switch(entity.category) {
+    case "ForSale":
+      createEntity('ForSaleListing', entity)
+    case "Housing":
+      createEntity('HousingListing', entity)
+    default:
+  }
+};
+
+/**
+ * The update function defines when reindexing should be triggered based on asynchronous
+ * events at the storage layer.
+ * @param  {[type]} web3          A client for interacting with an Ethereum storage backend.
+ * @param  {[type]} ipfs          A client for interacting with an IPFS storage backend.
+ * @param  {[type]} triggerUpdate A function to call anytime reindexing should occur.
+ */
+export const update = ({ ipfs, web3 }, triggerUpdate) => {
+  const abiCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ";
+  return ipfs.block
+    .get(abiCid)
+    .then(block => {
+      const abi = JSON.parse(block.data);
+      const OriginListing = web3.eth.contract(abi);
+      const originListing = OriginListing.at(
+        "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d"
+      );
+      const newListing = originListing.NewListing();
+      const listingPurchases = originListing.ListingPurchased();
+      newListing.watch((err) => {
+        if (!err) {
+          triggerUpdate()
+        }
+      })
+      listingUpdate.watch((err) => {
+        if (!err) {
+          triggerUpdate()
+        }
+      })
+    })
+}

specs/ingest-mapping/examples/OriginListing_old/EventSource-GenericDSL.yaml

@@ -0,0 +1,55 @@
+# Contract source: https://github.com/OriginProtocol/demo-dapp/blob/master/contracts/Listing.sol
+# specVersion indicates the version of the ingest mapping spec
+specVersion: 0.0.1
+# definitions are optional to help readability. May be used to specify aliases
+# for use later in the file. YAML requires that aliases be declared before they
+# are referenced.
+definitions:
+# Links to the GraphQL schema being mapped to
+schema:
+  # An IPLD link. This may not be needed if we have a single global schema.
+  /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k
+# A single DataSource may comprise multiple DataSets.
+# DataSets are composed of Data, Structure & Mapping, which are intentionally
+# decoupled.
+datasets:
+  # IPLD address of the data for the dataset
+  - data: eth-contract/0x06012c8cf97BEaD5deAe237070F9587f8E7A266d
+    structure:
+      format: eth-contract
+      abi:
+        # This is an IPLD link. Any value can be replaced with an IPLD link for
+        # improved legibility. See: https://github.com/ipld/specs/tree/master/ipld
+        /: /ipfs/QmUmg7BZC1YP1ca66rRtWKxpXp77WgVHrnv263JtDuvs2k
+    mapping:
+      strategy: eth-contract-event-source
+      # How to access entities using smart contract interface.
+      # Not the source of truth on current state, but used in event source logic.
+      accessors:
+        - entity: "@OriginProtocol/Listing"
+          # getListing is a public method on the OriginProtocol Listing contract
+          # for accessing the Listing entity
+          getByIndex: getListing
+      # This is where the core of the event sourcing logic happens using pipeline
+      # operators. The events being sourced are in the ethereum contract event log.
+      processEvents:
+        - fork:
+            - filter:
+                event: NewListing
+            - getEntityByIndex:
+                entity: "@OriginProtocol/Listing"
+                idField: id
+            - createEntity:
+                entity: "@OriginProtocol/Listing"
+                idField: id
+        - fork:
+            - filter:
+                event: ListingPurchased
+            # Gets the state *after* the Listing was updated.
+            - getEntityByIndex:
+                entity: "@OriginProtocol/Listing"
+                index: _index
+              # Updates an entity in The Graph by ID
+            - updateEntity:
+                entity: "@OriginProtocol/Listing"
+                idField: id

specs/ingest-mapping/examples/OriginListing_old/StateSnapshot-SpecializedDSL.yaml

@@ -0,0 +1,153 @@
+specVersion: mapping/v1
+mappings:
+  - kind: ethereum/array-collection
+    entity: ForSaleListing
+
+    # Collection name and size define the state variables or
+    # functions that provide the array collection and its length
+    collectionName: listings
+    collectionSize: listingsLength
+
+    # Fields defines fields that can be retrieved from the
+    # structs returned for each item in the array collection;
+    #
+    # Fields starting with an underscore (e.g. _ipfsHash) are
+    # considered internal and are stripped of before passing
+    # entities on to the indexer's data source event handler
+    fields:
+      lister:
+        kind: ethereum/struct-field-by-index
+        index: 0
+      _ipfsHash:
+        kind: ethereum/struct-field-by-index
+        index: 1
+      price:
+        kind: ethereum/struct-field-by-index
+        index: 2
+      unitsAvailable:
+        kind: ethereum/struct-field-by-index
+        index: 3
+
+    # Extra fields are additional fields that can come from additional
+    # smart contract function calls, external files e.g. on IPFS and more;
+    # they can refer to values of regular fields
+    extraFields:
+      id:
+        kind: ethereum/array-collection-item-index
+      _schema:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "schema"]
+      category:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "category"]
+      description:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "description"]
+
+    # Transformations are applied to entities after all of their
+    # fields have been collected; in this example, these items
+    # are filtered based on a filter called is-for-sale-listing, which
+    # is defined further below
+    transformations:
+      - kind: filter
+        name: is-for-sale-listing
+
+    # Events describe events that can lead to entities being reindexed
+    events:
+      - name: NewListing
+        pipeline:
+          - kind: ethereum/event-param
+            name: _index
+          - kind: ethereum/array-collection-item
+          - kind: filter
+            name: is-for-sale-listing
+          - kind: update-entity
+      - name: ListingPurchased
+        pipeline:
+          - kind: ethereum/event-param
+            name: _index
+          - kind: ethereum/array-collection-item
+          - kind: filter
+            name: is-for-sale-listing
+          - kind: update-entity
+
+    # Filters can be used in transformations and events and are defined
+    # in a single place
+    filters:
+      is-for-sale-listing:
+        kind: filter-by-field-value
+        field: _schema
+        value: https://localhost:3000/schemas/for-sale.json
+
+  - kind: ethereum/array-collection
+    entity: AnnouncementListing
+    collectionName: listings
+    collectionSize: listingsLength
+    fields:
+      lister:
+        kind: ethereum/struct-field-by-index
+        index: 0
+      _ipfsHash:
+        kind: ethereum/struct-field-by-index
+        index: 1
+      price:
+        kind: ethereum/struct-field-by-index
+        index: 2
+      unitsAvailable:
+        kind: ethereum/struct-field-by-index
+        index: 3
+    extraFields:
+      id:
+        kind: ethereum/array-collection-item-index
+      _schema:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "schema"]
+      category:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "category"]
+      description:
+        kind: ipfs/field-in-json-file
+        hash:
+          source: field
+          name: _ipfsHash
+        path: ["data", "description"]
+    transformations:
+      - kind: filter
+        name: filter-announcement
+    events:
+      - name: NewListing
+        pipeline:
+          - kind: ethereum/event-param
+            name: _index
+          - kind: ethereum/array-collection-item
+          - kind: filter
+            name: is-announcement-listing
+          - kind: update-entity
+      - name: ListingPurchased
+        pipeline:
+          - kind: ethereum/event-param
+            name: _index
+          - kind: ethereum/array-collection-item
+          - kind: filter
+            name: is-announcement-listing
+          - kind: update-entity
+    filters:
+      is-announcement-listing:
+        kind: filter-by-field-value
+        field: _schema
+        value: https://localhost:3000/schemas/announcement.json