feat: move to new inscription_feed predicate (#41)

* chore: move to inscription_feed * fix: change columns from bigint to numeric * fix: update schemas * fix: ignore events in the distant future * fix: ignore transfers in the future * chore: log when we insert out of order inscription * fix: address and offset optional * chore: adjust logs * test: fix all tests
hirosystems · Apr 18, 2023 · efa4a62 · efa4a62
1 parent 30d5d27
commit efa4a62
Show file tree

Hide file tree

Showing 14 changed files with 385 additions and 418 deletions.
diff --git a/migrations/1676395230930_inscriptions.ts b/migrations/1676395230930_inscriptions.ts
@@ -34,7 +34,7 @@ export function up(pgm: MigrationBuilder): void {
       notNull: true,
     },
     fee: {
-      type: 'bigint',
+      type: 'numeric',
       notNull: true,
     },
   });

diff --git a/migrations/1677284495299_locations.ts b/migrations/1677284495299_locations.ts
@@ -27,22 +27,19 @@ export function up(pgm: MigrationBuilder): void {
     },
     address: {
       type: 'text',
-      notNull: true,
     },
     output: {
       type: 'text',
       notNull: true,
     },
     offset: {
-      type: 'bigint',
-      notNull: true,
+      type: 'numeric',
     },
     value: {
-      type: 'bigint',
-      notNull: true,
+      type: 'numeric',
     },
     sat_ordinal: {
-      type: 'bigint',
+      type: 'numeric',
       notNull: true,
     },
     sat_rarity: {

diff --git a/src/api/schemas.ts b/src/api/schemas.ts
@@ -38,6 +38,8 @@ export const OpenApiSchemaOptions: SwaggerOptions = {
 // Parameters
 // ==========================
 
+const Nullable = <T extends TSchema>(type: T) => Type.Union([type, Type.Null()]);
+
 export const AddressParam = Type.String({
   title: 'Address',
   description: 'Bitcoin address',
@@ -208,9 +210,11 @@ export const InscriptionResponse = Type.Object({
     examples: ['1463d48e9248159084929294f64bda04487503d30ce7ab58365df1dc6fd58218i0'],
   }),
   number: Type.Integer({ examples: [248751] }),
-  address: Type.String({
-    examples: ['bc1pvwh2dl6h388x65rqq47qjzdmsqgkatpt4hye6daf7yxvl0z3xjgq247aq8'],
-  }),
+  address: Nullable(
+    Type.String({
+      examples: ['bc1pvwh2dl6h388x65rqq47qjzdmsqgkatpt4hye6daf7yxvl0z3xjgq247aq8'],
+    })
+  ),
   genesis_address: Type.String({
     examples: ['bc1pvwh2dl6h388x65rqq47qjzdmsqgkatpt4hye6daf7yxvl0z3xjgq247aq8'],
   }),
@@ -232,8 +236,8 @@ export const InscriptionResponse = Type.Object({
   output: Type.String({
     examples: ['1463d48e9248159084929294f64bda04487503d30ce7ab58365df1dc6fd58218:0'],
   }),
-  value: Type.String({ examples: ['546'] }),
-  offset: Type.String({ examples: ['0'] }),
+  value: Nullable(Type.String({ examples: ['546'] })),
+  offset: Nullable(Type.String({ examples: ['0'] })),
   sat_ordinal: Type.String({ examples: ['1232735286933201'] }),
   sat_rarity: Type.String({ examples: ['common'] }),
   sat_coinbase_height: Type.Integer({ examples: [430521] }),
@@ -274,9 +278,11 @@ export const InscriptionLocationResponse = Type.Object({
   block_hash: Type.String({
     examples: ['0000000000000000000452773967cdd62297137cdaf79950c5e8bb0c62075133'],
   }),
-  address: Type.String({
-    examples: ['bc1pvwh2dl6h388x65rqq47qjzdmsqgkatpt4hye6daf7yxvl0z3xjgq247aq8'],
-  }),
+  address: Nullable(
+    Type.String({
+      examples: ['bc1pvwh2dl6h388x65rqq47qjzdmsqgkatpt4hye6daf7yxvl0z3xjgq247aq8'],
+    })
+  ),
   tx_id: Type.String({
     examples: ['1463d48e9248159084929294f64bda04487503d30ce7ab58365df1dc6fd58218'],
   }),
@@ -286,8 +292,8 @@ export const InscriptionLocationResponse = Type.Object({
   output: Type.String({
     examples: ['1463d48e9248159084929294f64bda04487503d30ce7ab58365df1dc6fd58218:0'],
   }),
-  value: Type.String({ examples: ['546'] }),
-  offset: Type.String({ examples: ['0'] }),
+  value: Nullable(Type.String({ examples: ['546'] })),
+  offset: Nullable(Type.String({ examples: ['0'] })),
   timestamp: Type.Integer({ examples: [1677733170000] }),
 });
 export type InscriptionLocationResponse = Static<typeof InscriptionLocationResponse>;

diff --git a/src/api/util/helpers.ts b/src/api/util/helpers.ts
@@ -19,8 +19,8 @@ export function parseDbInscriptions(
     tx_id: i.tx_id,
     location: `${i.output}:${i.offset}`,
     output: i.output,
-    value: i.value.toString(),
-    offset: i.offset.toString(),
+    value: i.value,
+    offset: i.offset,
     sat_ordinal: i.sat_ordinal.toString(),
     sat_rarity: i.sat_rarity,
     sat_coinbase_height: i.sat_coinbase_height,
@@ -42,8 +42,8 @@ export function parseInscriptionLocations(items: DbLocation[]): InscriptionLocat
     tx_id: i.tx_id,
     location: `${i.output}:${i.offset}`,
     output: i.output,
-    value: i.value.toString(),
-    offset: i.offset.toString(),
+    value: i.value,
+    offset: i.offset,
     timestamp: i.timestamp.valueOf(),
   }));
 }

diff --git a/src/chainhook/helpers.ts b/src/chainhook/helpers.ts
@@ -4,126 +4,100 @@ import { PgStore } from '../pg/pg-store';
 import { ChainhookPayloadCType } from './schemas';
 
 /**
- * Process an `inscription_revealed` event from chainhooks and saves inscriptions to the DB.
+ * Process an `inscription_feed` event from chainhooks and saves inscription data to the DB.
  * @param payload - Event payload
  * @param db - DB
  */
-export async function processInscriptionRevealed(payload: unknown, db: PgStore): Promise<void> {
+export async function processInscriptionFeed(payload: unknown, db: PgStore): Promise<void> {
   if (!ChainhookPayloadCType.Check(payload)) {
     const errors = [...ChainhookPayloadCType.Errors(payload)];
-    logger.warn(errors, `[inscription_revealed] invalid payload`);
+    logger.error(errors, `[inscription_feed] invalid payload`);
     return;
   }
   for (const event of payload.rollback) {
     for (const tx of event.transactions) {
-      const reveal = tx.metadata.ordinal_operations[0].inscription_revealed;
-      if (!reveal) {
-        logger.warn(`[inscription_revealed] invalid rollback`);
-        continue;
+      for (const operation of tx.metadata.ordinal_operations) {
+        if (operation.inscription_revealed) {
+          const genesis_id = operation.inscription_revealed.inscription_id;
+          await db.rollBackInscriptionGenesis({ genesis_id });
+          logger.info(`[inscription_feed] rollback inscription ${genesis_id}`);
+        }
+        if (operation.inscription_transferred) {
+          const genesis_id = operation.inscription_transferred.inscription_id;
+          const satpoint = operation.inscription_transferred.satpoint_post_transfer.split(':');
+          const output = `${satpoint[0]}:${satpoint[1]}`;
+          await db.rollBackInscriptionTransfer({ genesis_id, output });
+          logger.info(`[inscription_feed] rollback transfer ${genesis_id} ${output}`);
+        }
       }
-      const genesis_id = reveal.inscription_id;
-      await db.rollBackInscriptionGenesis({ genesis_id });
-      logger.info(`[inscription_revealed] rollback inscription ${genesis_id}`);
     }
   }
   for (const event of payload.apply) {
     for (const tx of event.transactions) {
-      const reveal = tx.metadata.ordinal_operations[0].inscription_revealed;
-      if (!reveal) {
-        logger.warn(`[inscription_revealed] invalid apply`);
-        continue;
+      for (const operation of tx.metadata.ordinal_operations) {
+        if (operation.inscription_revealed) {
+          const reveal = operation.inscription_revealed;
+          const txId = tx.transaction_identifier.hash.substring(2);
+          const satoshi = new OrdinalSatoshi(reveal.ordinal_number);
+          await db.insertInscriptionGenesis({
+            inscription: {
+              genesis_id: reveal.inscription_id,
+              mime_type: reveal.content_type.split(';')[0],
+              content_type: reveal.content_type,
+              content_length: reveal.content_length,
+              number: reveal.inscription_number,
+              content: reveal.content_bytes,
+              fee: reveal.inscription_fee.toString(),
+            },
+            location: {
+              genesis_id: reveal.inscription_id,
+              block_height: event.block_identifier.index,
+              block_hash: event.block_identifier.hash.substring(2),
+              tx_id: txId,
+              address: reveal.inscriber_address,
+              output: `${txId}:0`,
+              offset: reveal.ordinal_offset.toString(),
+              value: reveal.inscription_output_value.toString(),
+              timestamp: event.timestamp,
+              sat_ordinal: reveal.ordinal_number.toString(),
+              sat_rarity: satoshi.rarity,
+              sat_coinbase_height: satoshi.blockHeight,
+            },
+          });
+          logger.info(
+            `[inscription_feed] apply inscription #${reveal.inscription_number} (${reveal.inscription_id}) at block ${event.block_identifier.index}`
+          );
+        }
+        if (operation.inscription_transferred) {
+          const transfer = operation.inscription_transferred;
+          const txId = tx.transaction_identifier.hash.substring(2);
+          const satpoint = transfer.satpoint_post_transfer.split(':');
+          const offset = satpoint[2];
+          const output = `${satpoint[0]}:${satpoint[1]}`;
+          const satoshi = new OrdinalSatoshi(transfer.ordinal_number);
+          await db.insertInscriptionTransfer({
+            location: {
+              genesis_id: transfer.inscription_id,
+              block_height: event.block_identifier.index,
+              block_hash: event.block_identifier.hash,
+              tx_id: txId,
+              address: transfer.updated_address,
+              output: output,
+              offset: offset ?? null,
+              value: transfer.post_transfer_output_value
+                ? transfer.post_transfer_output_value.toString()
+                : null,
+              timestamp: event.timestamp,
+              sat_ordinal: transfer.ordinal_number.toString(),
+              sat_rarity: satoshi.rarity,
+              sat_coinbase_height: satoshi.blockHeight,
+            },
+          });
+          logger.info(
+            `[inscription_feed] apply transfer for #${transfer.inscription_number} (${transfer.inscription_id}) to output ${output} at block ${event.block_identifier.index}`
+          );
+        }
       }
-      const txId = tx.transaction_identifier.hash.substring(2);
-      const utxo = tx.metadata.outputs[0];
-      const satoshi = new OrdinalSatoshi(reveal.ordinal_number);
-      await db.insertInscriptionGenesis({
-        inscription: {
-          genesis_id: reveal.inscription_id,
-          mime_type: reveal.content_type.split(';')[0],
-          content_type: reveal.content_type,
-          content_length: reveal.content_length,
-          number: reveal.inscription_number,
-          content: reveal.content_bytes,
-          fee: BigInt(reveal.inscription_fee),
-        },
-        location: {
-          genesis_id: reveal.inscription_id,
-          block_height: event.block_identifier.index,
-          block_hash: event.block_identifier.hash.substring(2),
-          tx_id: txId,
-          address: reveal.inscriber_address,
-          output: `${txId}:0`,
-          offset: BigInt(reveal.ordinal_offset),
-          value: BigInt(utxo.value),
-          timestamp: event.timestamp,
-          sat_ordinal: BigInt(reveal.ordinal_number),
-          sat_rarity: satoshi.rarity,
-          sat_coinbase_height: satoshi.blockHeight,
-        },
-      });
-      logger.info(
-        `[inscription_revealed] apply inscription #${reveal.inscription_number} (${reveal.inscription_id}) at block ${event.block_identifier.index}`
-      );
-    }
-  }
-}
-
-/**
- * Process an `inscription_transfer` event from chainhooks and saves new locations to the DB.
- * @param payload - Event payload
- * @param db - DB
- */
-export async function processInscriptionTransferred(payload: unknown, db: PgStore): Promise<void> {
-  if (!ChainhookPayloadCType.Check(payload)) {
-    const errors = [...ChainhookPayloadCType.Errors(payload)];
-    logger.warn(errors, `[inscription_transferred] invalid payload`);
-    return;
-  }
-  for (const event of payload.rollback) {
-    for (const tx of event.transactions) {
-      const transfer = tx.metadata.ordinal_operations[0].inscription_transferred;
-      if (!transfer) {
-        logger.warn(`[inscription_transferred] invalid rollback`);
-        continue;
-      }
-      const genesis_id = transfer.inscription_id;
-      const satpoint = transfer.satpoint_post_transfer.split(':');
-      const output = `${satpoint[0]}:${satpoint[1]}`;
-      await db.rollBackInscriptionTransfer({ genesis_id, output });
-      logger.info(`[inscription_transferred] rollback transfer ${genesis_id} ${output}`);
-    }
-  }
-  for (const event of payload.apply) {
-    for (const tx of event.transactions) {
-      const transfer = tx.metadata.ordinal_operations[0].inscription_transferred;
-      if (!transfer) {
-        logger.warn(`[inscription_transferred] invalid apply`);
-        continue;
-      }
-      const txId = tx.transaction_identifier.hash.substring(2);
-      const satpoint = transfer.satpoint_post_transfer.split(':');
-      const output = `${satpoint[0]}:${satpoint[1]}`;
-      const utxo = tx.metadata.outputs[0];
-      const satoshi = new OrdinalSatoshi(transfer.ordinal_number);
-      await db.insertInscriptionTransfer({
-        location: {
-          genesis_id: transfer.inscription_id,
-          block_height: event.block_identifier.index,
-          block_hash: event.block_identifier.hash,
-          tx_id: txId,
-          address: transfer.updated_address,
-          output: output,
-          offset: BigInt(satpoint[2]),
-          value: BigInt(utxo.value),
-          timestamp: event.timestamp,
-          sat_ordinal: BigInt(transfer.ordinal_number),
-          sat_rarity: satoshi.rarity,
-          sat_coinbase_height: satoshi.blockHeight,
-        },
-      });
-      logger.info(
-        `[inscription_transferred] apply transfer for #${transfer.inscription_number} (${transfer.inscription_id}) to output ${output} at block ${event.block_identifier.index}`
-      );
     }
   }
 }
diff --git a/src/chainhook/schemas.ts b/src/chainhook/schemas.ts
@@ -15,6 +15,7 @@ const InscriptionRevealed = Type.Object({
   inscription_number: Type.Integer(),
   inscription_fee: Type.Integer(),
   inscription_id: Type.String(),
+  inscription_output_value: Type.Integer(),
   inscriber_address: Type.String(),
   ordinal_number: Type.Integer(),
   ordinal_block_height: Type.Integer(),
@@ -26,9 +27,10 @@ const InscriptionTransferred = Type.Object({
   inscription_number: Type.Integer(),
   inscription_id: Type.String(),
   ordinal_number: Type.Integer(),
-  updated_address: Type.String(),
+  updated_address: Nullable(Type.String()),
   satpoint_pre_transfer: Type.String(),
   satpoint_post_transfer: Type.String(),
+  post_transfer_output_value: Nullable(Type.Integer()),
 });
 
 const OrdinalOperation = Type.Object({
@@ -46,7 +48,7 @@ const Transaction = Type.Object({
   operations: Type.Array(Type.Any()),
   metadata: Type.Object({
     ordinal_operations: Type.Array(OrdinalOperation),
-    outputs: Type.Array(Output),
+    outputs: Type.Optional(Type.Array(Output)),
     proof: Nullable(Type.String()),
   }),
 });
@@ -66,7 +68,7 @@ const ChainhookPayload = Type.Object({
     uuid: Type.String(),
     predicate: Type.Object({
       scope: Type.String(),
-      ordinal: Type.String(),
+      operation: Type.String(),
     }),
   }),
 });