fix: ordhook ingestion (#356)

* chore: launch configs * fix: migrations * fix: debug server * fix: optimize address handling * chore: upgrade client * fix: debug server * fix: disable gap check during replay mode * fix: adjust insert batch size * fix: debug server name * fix: batch insert owners * fix: optimize fks * fix: remove directly * fix: fk name * chore: drop unnecessary fks * fix: batch inscription transfers * fix: use full index for block tx index
hirosystems · Jun 11, 2024 · dfc003e · dfc003e
1 parent 1515b5d
commit dfc003e
Show file tree

Hide file tree

Showing 15 changed files with 123 additions and 128 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -14,6 +14,7 @@
       ],
       "outputCapture": "std",
       "internalConsoleOptions": "openOnSessionStart",
+      "envFile": "${workspaceFolder}/.env",
       "env": {
         "NODE_ENV": "development",
         "TS_NODE_SKIP_IGNORE": "true"
@@ -33,6 +34,7 @@
       ],
       "outputCapture": "std",
       "internalConsoleOptions": "openOnSessionStart",
+      "envFile": "${workspaceFolder}/.env",
       "env": {
         "NODE_ENV": "development",
         "TS_NODE_SKIP_IGNORE": "true",
@@ -53,13 +55,33 @@
       ],
       "outputCapture": "std",
       "internalConsoleOptions": "openOnSessionStart",
+      "envFile": "${workspaceFolder}/.env",
       "env": {
         "NODE_ENV": "development",
         "TS_NODE_SKIP_IGNORE": "true",
         "RUN_MODE": "writeonly"
       },
       "killBehavior": "polite",
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Run: debug server",
+      "runtimeArgs": [
+        "-r",
+        "ts-node/register"
+      ],
+      "args": [
+        "${workspaceFolder}/util/debug-server.ts"
+      ],
+      "outputCapture": "std",
+      "internalConsoleOptions": "openOnSessionStart",
+      "env": {
+        "NODE_ENV": "development",
+        "TS_NODE_SKIP_IGNORE": "true",
+      },
+      "killBehavior": "polite",
+    },
     {
       "type": "node",
       "request": "launch",

diff --git a/migrations/1676395230930_inscriptions.ts b/migrations/1676395230930_inscriptions.ts
@@ -31,6 +31,7 @@ export function up(pgm: MigrationBuilder): void {
     },
     address: {
       type: 'text',
+      notNull: true,
     },
     mime_type: {
       type: 'text',
@@ -76,11 +77,6 @@ export function up(pgm: MigrationBuilder): void {
     },
   });
   pgm.createConstraint('inscriptions', 'inscriptions_number_unique', 'UNIQUE(number)');
-  pgm.createConstraint(
-    'inscriptions',
-    'inscriptions_ordinal_number_fk',
-    'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE'
-  );
   pgm.createIndex('inscriptions', ['mime_type']);
   pgm.createIndex('inscriptions', ['recursive']);
   pgm.createIndex('inscriptions', [
@@ -89,4 +85,5 @@ export function up(pgm: MigrationBuilder): void {
   ]);
   pgm.createIndex('inscriptions', ['address']);
   pgm.createIndex('inscriptions', [{ name: 'updated_at', sort: 'DESC' }]);
+  pgm.createIndex('inscriptions', ['ordinal_number']);
 }
diff --git a/migrations/1677284495299_locations.ts b/migrations/1677284495299_locations.ts
@@ -28,6 +28,7 @@ export function up(pgm: MigrationBuilder): void {
     },
     address: {
       type: 'text',
+      notNull: true,
     },
     output: {
       type: 'text',
@@ -57,11 +58,6 @@ export function up(pgm: MigrationBuilder): void {
   pgm.createConstraint('locations', 'locations_pkey', {
     primaryKey: ['ordinal_number', 'block_height', 'tx_index'],
   });
-  pgm.createConstraint(
-    'locations',
-    'locations_ordinal_number_fk',
-    'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE'
-  );
   pgm.createIndex('locations', ['output', 'offset']);
   pgm.createIndex('locations', ['timestamp']);
   pgm.createIndex('locations', [

diff --git a/migrations/1677284495500_current-locations.ts b/migrations/1677284495500_current-locations.ts
@@ -19,18 +19,10 @@ export function up(pgm: MigrationBuilder): void {
     },
     address: {
       type: 'text',
+      notNull: true,
     },
   });
-  pgm.createConstraint(
-    'current_locations',
-    'current_locations_locations_fk',
-    'FOREIGN KEY(ordinal_number, block_height, tx_index) REFERENCES locations(ordinal_number, block_height, tx_index) ON DELETE CASCADE'
-  );
-  pgm.createConstraint(
-    'locations',
-    'locations_satoshis_fk',
-    'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE'
-  );
   pgm.createIndex('current_locations', ['ordinal_number'], { unique: true });
   pgm.createIndex('current_locations', ['address']);
+  pgm.createIndex('current_locations', ['block_height', 'tx_index']);
 }
diff --git a/migrations/1677284495501_inscription-transfers.ts b/migrations/1677284495501_inscription-transfers.ts
@@ -37,16 +37,6 @@ export function up(pgm: MigrationBuilder): void {
   pgm.createConstraint('inscription_transfers', 'inscription_transfers_pkey', {
     primaryKey: ['block_height', 'block_transfer_index'],
   });
-  pgm.createConstraint(
-    'inscription_transfers',
-    'inscription_transfers_locations_fk',
-    'FOREIGN KEY(ordinal_number, block_height, tx_index) REFERENCES locations(ordinal_number, block_height, tx_index) ON DELETE CASCADE'
-  );
-  pgm.createConstraint(
-    'inscription_transfers',
-    'inscription_transfers_satoshis_fk',
-    'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE'
-  );
   pgm.createIndex('inscription_transfers', ['genesis_id']);
   pgm.createIndex('inscription_transfers', ['number']);
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -9,7 +9,7 @@
     "build": "rimraf ./dist && tsc --project tsconfig.build.json",
     "start": "node dist/src/index.js",
     "start-ts": "ts-node ./src/index.ts",
-    "start:debug-server": "node dist/util/debug-server.js",
+    "start:debug-server": "ts-node ./util/debug-server.ts",
     "test": "jest --runInBand",
     "test:brc-20": "npm run test -- ./tests/brc-20/",
     "test:api": "npm run test -- ./tests/api/",
@@ -57,7 +57,7 @@
     "@fastify/swagger": "^8.3.1",
     "@fastify/type-provider-typebox": "^3.2.0",
     "@hirosystems/api-toolkit": "^1.4.0",
-    "@hirosystems/chainhook-client": "^1.8.0",
+    "@hirosystems/chainhook-client": "^1.10.0",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/commit-analyzer": "^10.0.4",
     "@semantic-release/git": "^10.0.1",

diff --git a/src/env.ts b/src/env.ts
@@ -33,9 +33,9 @@ const schema = Type.Object({
   ORDHOOK_NODE_RPC_PORT: Type.Number({ default: 20456, minimum: 0, maximum: 65535 }),
   /**
    * Authorization token that the ordhook node must send with every event to make sure it's
-   * coming from the valid instance
+   * coming from the valid instance. Leave it undefined if you wish to avoid header validation.
    */
-  ORDHOOK_NODE_AUTH_TOKEN: Type.String(),
+  ORDHOOK_NODE_AUTH_TOKEN: Type.Optional(Type.String()),
   /**
    * Register ordhook predicates automatically when the API is first launched. Set this to `false`
    * if you're configuring your predicates manually for any reason.
@@ -53,6 +53,8 @@ const schema = Type.Object({
     { default: 'default', replay: 'replay' },
     { default: 'default' }
   ),
+  /** If the API should automatically shut down when Ordhook ingestion mode is `replay` */
+  ORDHOOK_REPLAY_INGESTION_MODE_AUTO_SHUTDOWN: Type.Boolean({ default: true }),
 
   PGHOST: Type.String(),
   PGPORT: Type.Number({ default: 5432, minimum: 0, maximum: 65535 }),

diff --git a/src/ordhook/server.ts b/src/ordhook/server.ts
@@ -45,10 +45,11 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise<Chainho
   const serverOpts: ServerOptions = {
     hostname: ENV.API_HOST,
     port: ENV.EVENT_PORT,
-    auth_token: ENV.ORDHOOK_NODE_AUTH_TOKEN,
+    auth_token: ENV.ORDHOOK_NODE_AUTH_TOKEN ?? '',
     external_base_url: `http://${ENV.EXTERNAL_HOSTNAME}`,
     wait_for_chainhook_node: ENV.ORDHOOK_AUTO_PREDICATE_REGISTRATION,
-    validate_chainhook_payloads: true,
+    validate_chainhook_payloads: false,
+    validate_token_authorization: ENV.ORDHOOK_NODE_AUTH_TOKEN != undefined,
     body_limit: ENV.EVENT_SERVER_BODY_LIMIT,
     node_type: 'ordhook',
   };
@@ -58,7 +59,11 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise<Chainho
   const server = new ChainhookEventObserver(serverOpts, ordhookOpts);
   await server.start(predicates, async (uuid: string, payload: Payload) => {
     const streamed = payload.chainhook.is_streaming_blocks;
-    if (ENV.ORDHOOK_INGESTION_MODE === 'replay' && streamed) {
+    if (
+      ENV.ORDHOOK_INGESTION_MODE === 'replay' &&
+      ENV.ORDHOOK_REPLAY_INGESTION_MODE_AUTO_SHUTDOWN &&
+      streamed
+    ) {
       logger.info(`OrdhookServer finished replaying blocks, shutting down`);
       return shutdown();
     }
@@ -67,5 +72,7 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise<Chainho
     );
     await args.db.updateInscriptions(payload as BitcoinPayload);
   });
+  const chainTip = await args.db.getChainTipBlockHeight();
+  logger.info(`OrdhookServer chain tip is at ${chainTip}`);
   return server;
 }
diff --git a/src/pg/block-cache.ts b/src/pg/block-cache.ts
@@ -61,7 +61,7 @@ export class BlockCache {
       content_length: reveal.content_length,
       block_height: this.blockHeight,
       tx_index: reveal.tx_index,
-      address: reveal.inscriber_address,
+      address: reveal.inscriber_address ?? '',
       number: reveal.inscription_number.jubilee,
       classic_number: reveal.inscription_number.classic,
       content: removeNullBytes(reveal.content_bytes),
@@ -73,7 +73,8 @@ export class BlockCache {
       parent: reveal.parent,
       timestamp: this.timestamp,
     });
-    this.revealedNumbers.push(reveal.inscription_number.jubilee);
+    if (reveal.inscription_number.jubilee > 0)
+      this.revealedNumbers.push(reveal.inscription_number.jubilee);
     this.increaseMimeTypeCount(mime_type);
     this.increaseSatRarityCount(satoshi.rarity);
     this.increaseInscriptionTypeCount(reveal.inscription_number.classic < 0 ? 'cursed' : 'blessed');
@@ -85,7 +86,7 @@ export class BlockCache {
       tx_id,
       tx_index: reveal.tx_index,
       ordinal_number,
-      address: reveal.inscriber_address,
+      address: reveal.inscriber_address ?? '',
       output: `${satpoint.tx_id}:${satpoint.vout}`,
       offset: satpoint.offset ?? null,
       prev_output: null,
@@ -98,7 +99,7 @@ export class BlockCache {
       ordinal_number,
       block_height: this.blockHeight,
       tx_index: reveal.tx_index,
-      address: reveal.inscriber_address,
+      address: reveal.inscriber_address ?? '',
     });
     if (recursive_refs.length > 0) this.recursiveRefs.set(reveal.inscription_id, recursive_refs);
   }
@@ -107,7 +108,7 @@ export class BlockCache {
     const satpoint = parseSatPoint(transfer.satpoint_post_transfer);
     const prevSatpoint = parseSatPoint(transfer.satpoint_pre_transfer);
     const ordinal_number = transfer.ordinal_number.toString();
-    const address = transfer.destination.value ?? null;
+    const address = transfer.destination.value ?? '';
     this.locations.push({
       block_hash: this.blockHash,
       block_height: this.blockHeight,