fix: cache inscription counts for mime_type and sat_rarity (#55)

* chore: upgrade postgres

* chore: add materialized view migrations

* feat: consider cached counts

* fix: unified update transaction

* test: generator

* test: status

* test: server

* fix: remaining tests

migrations/1683047918926_mime-type-counts.ts

@@ -0,0 +1,12 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.createMaterializedView(
+    'mime_type_counts',
+    { data: true },
+    `SELECT mime_type, COUNT(*) AS count FROM inscriptions GROUP BY mime_type`
+  );
+}

migrations/1683061444855_sat-rarity-counts.ts

@@ -0,0 +1,18 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.createMaterializedView(
+    'sat_rarity_counts',
+    { data: true },
+    `
+    SELECT sat_rarity, COUNT(*) AS count
+    FROM inscriptions AS i
+    INNER JOIN locations AS loc ON loc.inscription_id = i.id
+    WHERE loc.current = TRUE
+    GROUP BY sat_rarity
+    `
+  );
+}

migrations/1683130423352_inscription-count.ts

@@ -0,0 +1,12 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.createMaterializedView(
+    'inscription_count',
+    { data: true },
+    `SELECT COUNT(*) AS count FROM inscriptions`
+  );
+}

migrations/1683130430977_chain-tip-view.ts

@@ -0,0 +1,38 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.dropTable('chain_tip');
+  pgm.createMaterializedView(
+    'chain_tip',
+    { data: true },
+    `SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations`
+  );
+}
+
+export function down(pgm: MigrationBuilder): void {
+  pgm.dropMaterializedView('chain_tip');
+  pgm.createTable('chain_tip', {
+    id: {
+      type: 'bool',
+      primaryKey: true,
+      default: true,
+    },
+    block_height: {
+      type: 'int',
+      notNull: true,
+      default: 767430, // First inscription block height
+    },
+    inscription_count: {
+      type: 'int',
+      notNull: true,
+      default: 0,
+    },
+  });
+  // Ensure only a single row can exist
+  pgm.addConstraint('chain_tip', 'chain_tip_one_row', 'CHECK(id)');
+  // Create the single row
+  pgm.sql('INSERT INTO chain_tip VALUES(DEFAULT)');
+}

package.json

@@ -57,7 +57,7 @@
     "fastify-metrics": "^10.2.0",
     "node-pg-migrate": "^6.2.2",
     "pino": "^8.10.0",
-    "postgres": "^3.3.3",
+    "postgres": "^3.3.4",
     "undici": "^5.8.0"
   }
 }

src/api/util/helpers.ts

@@ -66,3 +66,7 @@ export function hexToBuffer(hex: string): Buffer {
 }
 
 export const has0xPrefix = (id: string) => id.substr(0, 2).toLowerCase() === '0x';
+
+export function normalizedHexString(hex: string): string {
+  return has0xPrefix(hex) ? hex.substring(2) : hex;
+}

src/chainhook/helpers.ts

@@ -1,4 +1,3 @@
-import { OrdinalSatoshi } from '../api/util/ordinal-satoshi';
 import { logger } from '../logger';
 import { PgStore } from '../pg/pg-store';
 import { ChainhookPayloadCType } from './schemas';
@@ -14,91 +13,5 @@ export async function processInscriptionFeed(payload: unknown, db: PgStore): Pro
     logger.error(errors, `[inscription_feed] invalid payload`);
     return;
   }
-  for (const event of payload.rollback) {
-    for (const tx of event.transactions) {
-      for (const operation of tx.metadata.ordinal_operations) {
-        if (operation.inscription_revealed) {
-          const genesis_id = operation.inscription_revealed.inscription_id;
-          await db.rollBackInscriptionGenesis({ genesis_id });
-          logger.info(`[inscription_feed] rollback inscription ${genesis_id}`);
-        }
-        if (operation.inscription_transferred) {
-          const genesis_id = operation.inscription_transferred.inscription_id;
-          const satpoint = operation.inscription_transferred.satpoint_post_transfer.split(':');
-          const output = `${satpoint[0]}:${satpoint[1]}`;
-          await db.rollBackInscriptionTransfer({ genesis_id, output });
-          logger.info(`[inscription_feed] rollback transfer ${genesis_id} ${output}`);
-        }
-      }
-    }
-  }
-  for (const event of payload.apply) {
-    for (const tx of event.transactions) {
-      for (const operation of tx.metadata.ordinal_operations) {
-        if (operation.inscription_revealed) {
-          const reveal = operation.inscription_revealed;
-          const txId = tx.transaction_identifier.hash.substring(2);
-          const satoshi = new OrdinalSatoshi(reveal.ordinal_number);
-          await db.insertInscriptionGenesis({
-            inscription: {
-              genesis_id: reveal.inscription_id,
-              mime_type: reveal.content_type.split(';')[0],
-              content_type: reveal.content_type,
-              content_length: reveal.content_length,
-              number: reveal.inscription_number,
-              content: reveal.content_bytes,
-              fee: reveal.inscription_fee.toString(),
-            },
-            location: {
-              genesis_id: reveal.inscription_id,
-              block_height: event.block_identifier.index,
-              block_hash: event.block_identifier.hash.substring(2),
-              tx_id: txId,
-              address: reveal.inscriber_address,
-              output: `${txId}:0`,
-              offset: reveal.ordinal_offset.toString(),
-              value: reveal.inscription_output_value.toString(),
-              timestamp: event.timestamp,
-              sat_ordinal: reveal.ordinal_number.toString(),
-              sat_rarity: satoshi.rarity,
-              sat_coinbase_height: satoshi.blockHeight,
-            },
-          });
-          logger.info(
-            `[inscription_feed] apply inscription #${reveal.inscription_number} (${reveal.inscription_id}) at block ${event.block_identifier.index}`
-          );
-        }
-        if (operation.inscription_transferred) {
-          const transfer = operation.inscription_transferred;
-          const txId = tx.transaction_identifier.hash.substring(2);
-          const satpoint = transfer.satpoint_post_transfer.split(':');
-          const offset = satpoint[2];
-          const output = `${satpoint[0]}:${satpoint[1]}`;
-          const satoshi = new OrdinalSatoshi(transfer.ordinal_number);
-          await db.insertInscriptionTransfer({
-            location: {
-              genesis_id: transfer.inscription_id,
-              block_height: event.block_identifier.index,
-              block_hash: event.block_identifier.hash,
-              tx_id: txId,
-              address: transfer.updated_address,
-              output: output,
-              offset: offset ?? null,
-              value: transfer.post_transfer_output_value
-                ? transfer.post_transfer_output_value.toString()
-                : null,
-              timestamp: event.timestamp,
-              sat_ordinal: transfer.ordinal_number.toString(),
-              sat_rarity: satoshi.rarity,
-              sat_coinbase_height: satoshi.blockHeight,
-            },
-          });
-          logger.info(
-            `[inscription_feed] apply transfer for #${transfer.inscription_number} (${transfer.inscription_id}) to output ${output} at block ${event.block_identifier.index}`
-          );
-        }
-      }
-    }
-  }
-  await db.updateChainTipInscriptionCount();
+  await db.updateInscriptions(payload);
 }

src/chainhook/schemas.ts

@@ -8,7 +8,7 @@ const Block = Type.Object({
   hash: Type.String(),
 });
 
-const InscriptionRevealed = Type.Object({
+const InscriptionRevealedSchema = Type.Object({
   content_bytes: Type.String(),
   content_type: Type.String(),
   content_length: Type.Integer(),
@@ -22,8 +22,9 @@ const InscriptionRevealed = Type.Object({
   ordinal_offset: Type.Integer(),
   satpoint_post_inscription: Type.String(),
 });
+export type InscriptionRevealed = Static<typeof InscriptionRevealedSchema>;
 
-const InscriptionTransferred = Type.Object({
+const InscriptionTransferredSchema = Type.Object({
   inscription_number: Type.Integer(),
   inscription_id: Type.String(),
   ordinal_number: Type.Integer(),
@@ -32,18 +33,19 @@ const InscriptionTransferred = Type.Object({
   satpoint_post_transfer: Type.String(),
   post_transfer_output_value: Nullable(Type.Integer()),
 });
+export type InscriptionTransferred = Static<typeof InscriptionTransferredSchema>;
 
 const OrdinalOperation = Type.Object({
-  inscription_revealed: Type.Optional(InscriptionRevealed),
-  inscription_transferred: Type.Optional(InscriptionTransferred),
+  inscription_revealed: Type.Optional(InscriptionRevealedSchema),
+  inscription_transferred: Type.Optional(InscriptionTransferredSchema),
 });
 
 const Output = Type.Object({
   script_pubkey: Type.String(),
   value: Type.Integer(),
 });
 
-const Transaction = Type.Object({
+const TransactionSchema = Type.Object({
   transaction_identifier: Type.Object({ hash: Type.String() }),
   operations: Type.Array(Type.Any()),
   metadata: Type.Object({
@@ -52,14 +54,16 @@ const Transaction = Type.Object({
     proof: Nullable(Type.String()),
   }),
 });
+export type Transaction = Static<typeof TransactionSchema>;
 
 const Event = Type.Object({
   block_identifier: Block,
   parent_block_identifier: Block,
   timestamp: Type.Integer(),
-  transactions: Type.Array(Transaction),
+  transactions: Type.Array(TransactionSchema),
   metadata: Type.Any(),
 });
+export type InscriptionEvent = Static<typeof Event>;
 
 const ChainhookPayload = Type.Object({
   apply: Type.Array(Event),

src/pg/helpers.ts

@@ -1,7 +1,11 @@
 import { Static, Type } from '@sinclair/typebox';
 import { TypeCompiler } from '@sinclair/typebox/compiler';
 import { hexToBuffer } from '../api/util/helpers';
-import { DbInscriptionInsert } from './types';
+import {
+  DbInscriptionIndexFilters,
+  DbInscriptionIndexResultCountType,
+  DbInscriptionInsert,
+} from './types';
 
 const OpJson = Type.Object(
   {
@@ -32,3 +36,26 @@ export function inscriptionContentToJson(inscription: DbInscriptionInsert): OpJs
     }
   }
 }
+
+/**
+ * Returns which inscription count is required based on filters sent to the index endpoint.
+ * @param filters - DbInscriptionIndexFilters
+ * @returns DbInscriptionIndexResultCountType
+ */
+export function getIndexResultCountType(
+  filters?: DbInscriptionIndexFilters
+): DbInscriptionIndexResultCountType {
+  if (!filters) return DbInscriptionIndexResultCountType.all;
+  // Remove undefined values.
+  Object.keys(filters).forEach(
+    key =>
+      filters[key as keyof DbInscriptionIndexFilters] === undefined &&
+      delete filters[key as keyof DbInscriptionIndexFilters]
+  );
+  // Check for selected filter.
+  if (Object.keys(filters).length === 1) {
+    if (filters.mime_type) return DbInscriptionIndexResultCountType.mimeType;
+    if (filters.sat_rarity) return DbInscriptionIndexResultCountType.satRarity;
+  }
+  return DbInscriptionIndexResultCountType.custom;
+}