From 97874cc1461d4e321d5143c70d68927ace62eec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Wed, 5 Jul 2023 15:00:18 -0600 Subject: [PATCH] feat!: optimize transfer replay capability (#129) * chore: first migration * chore: first pass * refactor: consolidate migrations * test: replay capability * fix: warn when inserting location with missing prev_output * chore: tweak transfer gap query --- migrations/1676395230930_inscriptions.ts | 24 +- migrations/1677284495299_locations.ts | 34 +- migrations/1677360299810_chain-tip.ts | 27 +- migrations/1679686636818_json-contents.ts | 4 +- ...82654536767_chain-tip-inscription-count.ts | 14 - migrations/1683042840697_expand-int-sizes.ts | 32 -- migrations/1683047918926_mime-type-counts.ts | 1 + migrations/1683061444855_sat-rarity-counts.ts | 3 +- migrations/1683130423352_inscription-count.ts | 1 + migrations/1683130430977_chain-tip-view.ts | 39 -- migrations/1685378650151_prev-location.ts | 16 - migrations/1686170300438_curse-type.ts | 12 - .../1688153654886_concurrent-view-indexes.ts | 11 - src/pg/pg-store.ts | 349 ++++++++---------- src/pg/types.ts | 28 +- tests/server.test.ts | 124 ++++++- 16 files changed, 328 insertions(+), 391 deletions(-) delete mode 100644 migrations/1682654536767_chain-tip-inscription-count.ts delete mode 100644 migrations/1683042840697_expand-int-sizes.ts delete mode 100644 migrations/1683130430977_chain-tip-view.ts delete mode 100644 migrations/1685378650151_prev-location.ts delete mode 100644 migrations/1686170300438_curse-type.ts delete mode 100644 migrations/1688153654886_concurrent-view-indexes.ts diff --git a/migrations/1676395230930_inscriptions.ts b/migrations/1676395230930_inscriptions.ts index 4d0dc6e0..e467026d 100644 --- a/migrations/1676395230930_inscriptions.ts +++ b/migrations/1676395230930_inscriptions.ts @@ -6,7 +6,7 @@ export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { pgm.createTable('inscriptions', { id: { - type: 'serial', + type: 'bigserial', primaryKey: true, }, genesis_id: { @@ -14,7 +14,19 @@ export function up(pgm: MigrationBuilder): void { notNull: true, }, number: { - type: 'int', + type: 'bigint', + notNull: true, + }, + sat_ordinal: { + type: 'numeric', + notNull: true, + }, + sat_rarity: { + type: 'text', + notNull: true, + }, + sat_coinbase_height: { + type: 'bigint', notNull: true, }, mime_type: { @@ -26,7 +38,7 @@ export function up(pgm: MigrationBuilder): void { notNull: true, }, content_length: { - type: 'int', + type: 'bigint', notNull: true, }, content: { @@ -37,8 +49,14 @@ export function up(pgm: MigrationBuilder): void { type: 'numeric', notNull: true, }, + curse_type: { + type: 'text', + }, }); pgm.createConstraint('inscriptions', 'inscriptions_number_unique', 'UNIQUE(number)'); pgm.createIndex('inscriptions', ['genesis_id']); pgm.createIndex('inscriptions', ['mime_type']); + pgm.createIndex('inscriptions', ['sat_ordinal']); + pgm.createIndex('inscriptions', ['sat_rarity']); + pgm.createIndex('inscriptions', ['sat_coinbase_height']); } diff --git a/migrations/1677284495299_locations.ts b/migrations/1677284495299_locations.ts index f277da6e..ec3bd90e 100644 --- a/migrations/1677284495299_locations.ts +++ b/migrations/1677284495299_locations.ts @@ -6,15 +6,18 @@ export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { pgm.createTable('locations', { id: { - type: 'serial', + type: 'bigserial', primaryKey: true, }, inscription_id: { - type: 'int', + type: 'bigint', + }, + genesis_id: { + type: 'text', notNull: true, }, block_height: { - type: 'int', + type: 'bigint', notNull: true, }, block_hash: { @@ -35,20 +38,14 @@ export function up(pgm: MigrationBuilder): void { offset: { type: 'numeric', }, - value: { - type: 'numeric', + prev_output: { + type: 'text', }, - sat_ordinal: { + prev_offset: { type: 'numeric', - notNull: true, }, - sat_rarity: { - type: 'text', - notNull: true, - }, - sat_coinbase_height: { - type: 'int', - notNull: true, + value: { + type: 'numeric', }, timestamp: { type: 'timestamptz', @@ -72,15 +69,14 @@ export function up(pgm: MigrationBuilder): void { ); pgm.createConstraint( 'locations', - 'locations_inscription_id_block_height_unique', - 'UNIQUE(inscription_id, block_height)' + 'locations_genesis_id_block_height_unique', + 'UNIQUE(genesis_id, block_height)' ); + pgm.createIndex('locations', ['genesis_id']); pgm.createIndex('locations', ['block_height']); pgm.createIndex('locations', ['block_hash']); pgm.createIndex('locations', ['address']); pgm.createIndex('locations', ['output']); - pgm.createIndex('locations', ['sat_ordinal']); - pgm.createIndex('locations', ['sat_rarity']); - pgm.createIndex('locations', ['sat_coinbase_height']); pgm.createIndex('locations', ['timestamp']); + pgm.createIndex('locations', ['prev_output']); } diff --git a/migrations/1677360299810_chain-tip.ts b/migrations/1677360299810_chain-tip.ts index aabffbf8..7dbab3d3 100644 --- a/migrations/1677360299810_chain-tip.ts +++ b/migrations/1677360299810_chain-tip.ts @@ -4,24 +4,11 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.createTable('chain_tip', { - id: { - type: 'bool', - primaryKey: true, - default: true, - }, - block_height: { - type: 'int', - notNull: true, - default: 767430, // First inscription block height - }, - }); - // Ensure only a single row can exist - pgm.addConstraint('chain_tip', 'chain_tip_one_row', 'CHECK(id)'); - // Create the single row - pgm.sql('INSERT INTO chain_tip VALUES(DEFAULT)'); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropTable('chain_tip'); + pgm.createMaterializedView( + 'chain_tip', + { data: true }, + // Set block height 767430 (inscription #0 genesis) as default. + `SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations` + ); + pgm.createIndex('chain_tip', ['block_height'], { unique: true }); } diff --git a/migrations/1679686636818_json-contents.ts b/migrations/1679686636818_json-contents.ts index 1e18722b..2dd29345 100644 --- a/migrations/1679686636818_json-contents.ts +++ b/migrations/1679686636818_json-contents.ts @@ -6,11 +6,11 @@ export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { pgm.createTable('json_contents', { id: { - type: 'serial', + type: 'bigserial', primaryKey: true, }, inscription_id: { - type: 'int', + type: 'bigint', notNull: true, }, p: { diff --git a/migrations/1682654536767_chain-tip-inscription-count.ts b/migrations/1682654536767_chain-tip-inscription-count.ts deleted file mode 100644 index bd86d607..00000000 --- a/migrations/1682654536767_chain-tip-inscription-count.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.addColumn('chain_tip', { - inscription_count: { - type: 'int', - notNull: true, - default: 0, - }, - }); -} diff --git a/migrations/1683042840697_expand-int-sizes.ts b/migrations/1683042840697_expand-int-sizes.ts deleted file mode 100644 index 7abd6d6b..00000000 --- a/migrations/1683042840697_expand-int-sizes.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.alterColumn('inscriptions', 'id', { type: 'bigint' }); - pgm.alterColumn('inscriptions', 'number', { type: 'bigint' }); - pgm.alterColumn('inscriptions', 'content_length', { type: 'bigint' }); - - pgm.alterColumn('locations', 'id', { type: 'bigint' }); - pgm.alterColumn('locations', 'inscription_id', { type: 'bigint' }); - pgm.alterColumn('locations', 'block_height', { type: 'bigint' }); - pgm.alterColumn('locations', 'sat_coinbase_height', { type: 'bigint' }); - - pgm.alterColumn('json_contents', 'id', { type: 'bigint' }); - pgm.alterColumn('json_contents', 'inscription_id', { type: 'bigint' }); -} - -export function down(pgm: MigrationBuilder): void { - pgm.alterColumn('inscriptions', 'id', { type: 'int' }); - pgm.alterColumn('inscriptions', 'number', { type: 'int' }); - pgm.alterColumn('inscriptions', 'content_length', { type: 'int' }); - - pgm.alterColumn('locations', 'id', { type: 'int' }); - pgm.alterColumn('locations', 'inscription_id', { type: 'int' }); - pgm.alterColumn('locations', 'block_height', { type: 'int' }); - pgm.alterColumn('locations', 'sat_coinbase_height', { type: 'int' }); - - pgm.alterColumn('json_contents', 'id', { type: 'int' }); - pgm.alterColumn('json_contents', 'inscription_id', { type: 'int' }); -} diff --git a/migrations/1683047918926_mime-type-counts.ts b/migrations/1683047918926_mime-type-counts.ts index c241455f..be4d52d1 100644 --- a/migrations/1683047918926_mime-type-counts.ts +++ b/migrations/1683047918926_mime-type-counts.ts @@ -9,4 +9,5 @@ export function up(pgm: MigrationBuilder): void { { data: true }, `SELECT mime_type, COUNT(*) AS count FROM inscriptions GROUP BY mime_type` ); + pgm.createIndex('mime_type_counts', ['mime_type'], { unique: true }); } diff --git a/migrations/1683061444855_sat-rarity-counts.ts b/migrations/1683061444855_sat-rarity-counts.ts index 43f2678f..921d59d0 100644 --- a/migrations/1683061444855_sat-rarity-counts.ts +++ b/migrations/1683061444855_sat-rarity-counts.ts @@ -10,9 +10,8 @@ export function up(pgm: MigrationBuilder): void { ` SELECT sat_rarity, COUNT(*) AS count FROM inscriptions AS i - INNER JOIN locations AS loc ON loc.inscription_id = i.id - WHERE loc.current = TRUE GROUP BY sat_rarity ` ); + pgm.createIndex('sat_rarity_counts', ['sat_rarity'], { unique: true }); } diff --git a/migrations/1683130423352_inscription-count.ts b/migrations/1683130423352_inscription-count.ts index 3799ab19..30c5b0fa 100644 --- a/migrations/1683130423352_inscription-count.ts +++ b/migrations/1683130423352_inscription-count.ts @@ -9,4 +9,5 @@ export function up(pgm: MigrationBuilder): void { { data: true }, `SELECT COUNT(*) AS count FROM inscriptions` ); + pgm.createIndex('inscription_count', ['count'], { unique: true }); } diff --git a/migrations/1683130430977_chain-tip-view.ts b/migrations/1683130430977_chain-tip-view.ts deleted file mode 100644 index d73ca3d0..00000000 --- a/migrations/1683130430977_chain-tip-view.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropTable('chain_tip'); - pgm.createMaterializedView( - 'chain_tip', - { data: true }, - // Set block height 767430 (inscription #0 genesis) as default. - `SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations` - ); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropMaterializedView('chain_tip'); - pgm.createTable('chain_tip', { - id: { - type: 'bool', - primaryKey: true, - default: true, - }, - block_height: { - type: 'int', - notNull: true, - default: 767430, // First inscription block height - }, - inscription_count: { - type: 'int', - notNull: true, - default: 0, - }, - }); - // Ensure only a single row can exist - pgm.addConstraint('chain_tip', 'chain_tip_one_row', 'CHECK(id)'); - // Create the single row - pgm.sql('INSERT INTO chain_tip VALUES(DEFAULT)'); -} diff --git a/migrations/1685378650151_prev-location.ts b/migrations/1685378650151_prev-location.ts deleted file mode 100644 index 721119b9..00000000 --- a/migrations/1685378650151_prev-location.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.addColumns('locations', { - prev_output: { - type: 'text', - }, - prev_offset: { - type: 'numeric', - }, - }); - pgm.createIndex('locations', ['prev_output']); -} diff --git a/migrations/1686170300438_curse-type.ts b/migrations/1686170300438_curse-type.ts deleted file mode 100644 index 0c9d873e..00000000 --- a/migrations/1686170300438_curse-type.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.addColumns('inscriptions', { - curse_type: { - type: 'text', - }, - }); -} diff --git a/migrations/1688153654886_concurrent-view-indexes.ts b/migrations/1688153654886_concurrent-view-indexes.ts deleted file mode 100644 index 4378d52a..00000000 --- a/migrations/1688153654886_concurrent-view-indexes.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createIndex('chain_tip', ['block_height'], { unique: true }); - pgm.createIndex('mime_type_counts', ['mime_type'], { unique: true }); - pgm.createIndex('sat_rarity_counts', ['sat_rarity'], { unique: true }); - pgm.createIndex('inscription_count', ['count'], { unique: true }); -} diff --git a/src/pg/pg-store.ts b/src/pg/pg-store.ts index 7925ff77..b9638ec5 100644 --- a/src/pg/pg-store.ts +++ b/src/pg/pg-store.ts @@ -59,24 +59,23 @@ export class PgStore extends BasePgStore { * @param args - Apply/Rollback Chainhook events */ async updateInscriptions(payload: Payload): Promise { - const updatedInscriptionIds = new Set(); + const updatedGenesisIds = new Set(); let updatedBlockHeightMin = Infinity; await this.sqlWriteTransaction(async sql => { for (const rollbackEvent of payload.rollback) { const event = rollbackEvent as BitcoinEvent; + const block_height = event.block_identifier.index; for (const tx of event.transactions) { for (const operation of tx.metadata.ordinal_operations) { if (operation.inscription_revealed) { const number = operation.inscription_revealed.inscription_number; const genesis_id = operation.inscription_revealed.inscription_id; - await this.rollBackInscriptionGenesis({ genesis_id }); - logger.info(`PgStore rollback reveal #${number} (${genesis_id})`); + await this.rollBackInscription({ genesis_id, number, block_height }); } if (operation.cursed_inscription_revealed) { const number = operation.cursed_inscription_revealed.inscription_number; const genesis_id = operation.cursed_inscription_revealed.inscription_id; - await this.rollBackInscriptionGenesis({ genesis_id }); - logger.info(`PgStore rollback cursed reveal #${number} (${genesis_id})`); + await this.rollBackInscription({ genesis_id, number, block_height }); } if (operation.inscription_transferred) { const genesis_id = operation.inscription_transferred.inscription_id; @@ -84,9 +83,8 @@ export class PgStore extends BasePgStore { operation.inscription_transferred.satpoint_post_transfer ); const output = `${satpoint.tx_id}:${satpoint.vout}`; - const id = await this.rollBackInscriptionTransfer({ genesis_id, output }); - if (id) updatedInscriptionIds.add(id); - logger.info(`PgStore rollback transfer (${genesis_id}) ${output}`); + await this.rollBackLocation({ genesis_id, output, block_height }); + updatedGenesisIds.add(genesis_id); } } } @@ -103,7 +101,7 @@ export class PgStore extends BasePgStore { const reveal = operation.inscription_revealed; const satoshi = new OrdinalSatoshi(reveal.ordinal_number); const satpoint = parseSatPoint(reveal.satpoint_post_inscription); - const id = await this.insertInscriptionGenesis({ + await this.insertInscription({ inscription: { genesis_id: reveal.inscription_id, mime_type: reveal.content_type.split(';')[0], @@ -113,6 +111,9 @@ export class PgStore extends BasePgStore { content: reveal.content_bytes, fee: reveal.inscription_fee.toString(), curse_type: null, + sat_ordinal: reveal.ordinal_number.toString(), + sat_rarity: satoshi.rarity, + sat_coinbase_height: satoshi.blockHeight, }, location: { block_hash, @@ -126,15 +127,9 @@ export class PgStore extends BasePgStore { prev_offset: null, value: reveal.inscription_output_value.toString(), timestamp: event.timestamp, - sat_ordinal: reveal.ordinal_number.toString(), - sat_rarity: satoshi.rarity, - sat_coinbase_height: satoshi.blockHeight, }, }); - if (id) updatedInscriptionIds.add(id); - logger.info( - `PgStore reveal #${reveal.inscription_number} (${reveal.inscription_id}) at block ${block_height}` - ); + updatedGenesisIds.add(reveal.inscription_id); } if (operation.cursed_inscription_revealed) { const reveal = operation.cursed_inscription_revealed; @@ -145,7 +140,7 @@ export class PgStore extends BasePgStore { ? reveal.curse_type : JSON.stringify(reveal.curse_type) : null; - const id = await this.insertInscriptionGenesis({ + await this.insertInscription({ inscription: { genesis_id: reveal.inscription_id, mime_type: reveal.content_type.split(';')[0], @@ -155,6 +150,9 @@ export class PgStore extends BasePgStore { content: reveal.content_bytes, fee: reveal.inscription_fee.toString(), curse_type, + sat_ordinal: reveal.ordinal_number.toString(), + sat_rarity: satoshi.rarity, + sat_coinbase_height: satoshi.blockHeight, }, location: { block_hash, @@ -168,64 +166,39 @@ export class PgStore extends BasePgStore { prev_offset: null, value: reveal.inscription_output_value.toString(), timestamp: event.timestamp, - sat_ordinal: reveal.ordinal_number.toString(), - sat_rarity: satoshi.rarity, - sat_coinbase_height: satoshi.blockHeight, }, }); - if (id) updatedInscriptionIds.add(id); - logger.info( - `PgStore cursed reveal #${reveal.inscription_number} (${reveal.inscription_id}) at block ${block_height}` - ); + updatedGenesisIds.add(reveal.inscription_id); } if (operation.inscription_transferred) { const transfer = operation.inscription_transferred; const satpoint = parseSatPoint(transfer.satpoint_post_transfer); const prevSatpoint = parseSatPoint(transfer.satpoint_pre_transfer); - const genesis = await this.getInscriptionGenesis({ - genesis_id: transfer.inscription_id, + await this.insertLocation({ + location: { + block_hash, + block_height, + tx_id, + genesis_id: transfer.inscription_id, + address: transfer.updated_address, + output: `${satpoint.tx_id}:${satpoint.vout}`, + offset: satpoint.offset ?? null, + prev_output: `${prevSatpoint.tx_id}:${prevSatpoint.vout}`, + prev_offset: prevSatpoint.offset ?? null, + value: transfer.post_transfer_output_value + ? transfer.post_transfer_output_value.toString() + : null, + timestamp: event.timestamp, + }, }); - if (genesis) { - const inscription_id = parseInt(genesis.inscription_id); - await this.insertInscriptionTransfer({ - inscription_id, - location: { - block_hash, - block_height, - tx_id, - genesis_id: transfer.inscription_id, - address: transfer.updated_address, - output: `${satpoint.tx_id}:${satpoint.vout}`, - offset: satpoint.offset ?? null, - prev_output: `${prevSatpoint.tx_id}:${prevSatpoint.vout}`, - prev_offset: prevSatpoint.offset ?? null, - value: transfer.post_transfer_output_value - ? transfer.post_transfer_output_value.toString() - : null, - timestamp: event.timestamp, - // TODO: Store these fields in `inscriptions` instead of `locations`. - sat_ordinal: genesis.sat_ordinal, - sat_rarity: genesis.sat_rarity, - sat_coinbase_height: parseInt(genesis.sat_coinbase_height), - }, - }); - updatedInscriptionIds.add(inscription_id); - logger.info( - `PgStore transfer (${transfer.inscription_id}) to output ${satpoint.tx_id}:${satpoint.vout} at block ${block_height}` - ); - } else { - logger.warn( - { block_height, genesis_id: transfer.inscription_id }, - `PgStore ignoring transfer for an inscription that does not exist` - ); - } + updatedGenesisIds.add(transfer.inscription_id); } } } updatedBlockHeightMin = Math.min(updatedBlockHeightMin, event.block_identifier.index); } }); - await this.normalizeInscriptionLocations({ inscription_id: Array.from(updatedInscriptionIds) }); + await this.normalizeInscriptionLocations({ genesis_id: Array.from(updatedGenesisIds) }); await this.refreshMaterializedView('chain_tip'); // Skip expensive view refreshes if we're not streaming any live blocks yet. if (payload.chainhook.is_streaming_blocks) { @@ -348,10 +321,10 @@ export class PgStore extends BasePgStore { let orderBy = sql`gen.block_height`; switch (sort?.order_by) { case OrderBy.ordinal: - orderBy = sql`loc.sat_ordinal`; + orderBy = sql`i.sat_ordinal`; break; case OrderBy.rarity: - orderBy = sql`ARRAY_POSITION(ARRAY['common','uncommon','rare','epic','legendary','mythic'], loc.sat_rarity)`; + orderBy = sql`ARRAY_POSITION(ARRAY['common','uncommon','rare','epic','legendary','mythic'], i.sat_rarity)`; break; } // `ORDER` statement @@ -365,6 +338,9 @@ export class PgStore extends BasePgStore { i.content_length, i.fee AS genesis_fee, i.curse_type, + i.sat_ordinal, + i.sat_rarity, + i.sat_coinbase_height, gen.block_height AS genesis_block_height, gen.block_hash AS genesis_block_hash, gen.tx_id AS genesis_tx_id, @@ -374,11 +350,8 @@ export class PgStore extends BasePgStore { loc.address, loc.output, loc.offset, - loc.sat_ordinal, - loc.sat_rarity, loc.timestamp, loc.value, - loc.sat_coinbase_height, ${ countType === DbInscriptionIndexResultCountType.custom ? sql`COUNT(*) OVER() as total` @@ -415,12 +388,12 @@ export class PgStore extends BasePgStore { } ${ filters?.from_sat_coinbase_height - ? sql`AND loc.sat_coinbase_height >= ${filters.from_sat_coinbase_height}` + ? sql`AND i.sat_coinbase_height >= ${filters.from_sat_coinbase_height}` : sql`` } ${ filters?.to_sat_coinbase_height - ? sql`AND loc.sat_coinbase_height <= ${filters.to_sat_coinbase_height}` + ? sql`AND i.sat_coinbase_height <= ${filters.to_sat_coinbase_height}` : sql`` } ${ @@ -435,10 +408,10 @@ export class PgStore extends BasePgStore { } ${ filters?.from_sat_ordinal - ? sql`AND loc.sat_ordinal >= ${filters.from_sat_ordinal}` + ? sql`AND i.sat_ordinal >= ${filters.from_sat_ordinal}` : sql`` } - ${filters?.to_sat_ordinal ? sql`AND loc.sat_ordinal <= ${filters.to_sat_ordinal}` : sql``} + ${filters?.to_sat_ordinal ? sql`AND i.sat_ordinal <= ${filters.to_sat_ordinal}` : sql``} ${filters?.number?.length ? sql`AND i.number IN ${sql(filters.number)}` : sql``} ${filters?.from_number ? sql`AND i.number >= ${filters.from_number}` : sql``} ${filters?.to_number ? sql`AND i.number <= ${filters.to_number}` : sql``} @@ -447,10 +420,10 @@ export class PgStore extends BasePgStore { ${filters?.output ? sql`AND loc.output = ${filters.output}` : sql``} ${ filters?.sat_rarity?.length - ? sql`AND loc.sat_rarity IN ${sql(filters.sat_rarity)}` + ? sql`AND i.sat_rarity IN ${sql(filters.sat_rarity)}` : sql`` } - ${filters?.sat_ordinal ? sql`AND loc.sat_ordinal = ${filters.sat_ordinal}` : sql``} + ${filters?.sat_ordinal ? sql`AND i.sat_ordinal = ${filters.sat_ordinal}` : sql``} ORDER BY ${orderBy} ${order} LIMIT ${page.limit} OFFSET ${page.offset} @@ -474,19 +447,6 @@ export class PgStore extends BasePgStore { }); } - async getInscriptionGenesis(args: { genesis_id: string }): Promise { - const results = await this.sql` - SELECT ${this.sql(LOCATIONS_COLUMNS.map(c => `l.${c}`))} - FROM locations AS l - INNER JOIN inscriptions AS i ON l.inscription_id = i.id - WHERE i.genesis_id = ${args.genesis_id} AND genesis = TRUE - LIMIT 1 - `; - if (results.count > 0) { - return results[0]; - } - } - async getInscriptionLocations( args: InscriptionIdentifier & { limit: number; offset: number } ): Promise> { @@ -607,65 +567,15 @@ export class PgStore extends BasePgStore { } ${this.sql(viewName)}`; } - private async insertInscriptionGenesis(args: { + private async insertInscription(args: { inscription: DbInscriptionInsert; location: DbLocationInsert; }): Promise { let inscription_id: number | undefined; await this.sqlWriteTransaction(async sql => { - // Are we upserting? - const prevInscription = await sql<{ id: number }[]>` + const upsert = await sql<{ id: number }[]>` SELECT id FROM inscriptions WHERE number = ${args.inscription.number} `; - if (prevInscription.count !== 0) { - logger.warn( - { - block_height: args.location.block_height, - genesis_id: args.inscription.genesis_id, - }, - `PgStore upserting inscription genesis #${args.inscription.number}` - ); - } else { - // Is this a sequential genesis insert? - if (args.inscription.number < 0) { - // Is it a cursed inscription? - const maxCursed = await this.getMaxCursedInscriptionNumber(); - if (maxCursed !== undefined && maxCursed - 1 !== args.inscription.number) { - logger.warn( - { - block_height: args.location.block_height, - genesis_id: args.inscription.genesis_id, - }, - `PgStore inserting out-of-order cursed inscription genesis #${args.inscription.number}, current max is #${maxCursed}` - ); - } - } else { - const maxNumber = await this.getMaxInscriptionNumber(); - if (maxNumber !== undefined && maxNumber + 1 !== args.inscription.number) { - logger.warn( - { - block_height: args.location.block_height, - genesis_id: args.inscription.genesis_id, - }, - `PgStore inserting out-of-order inscription genesis #${args.inscription.number}, current max is #${maxNumber}` - ); - } - // Is this a blessed inscription in a duplicate sat? - const dup = await sql<{ id: number }[]>` - SELECT id FROM locations WHERE sat_ordinal = ${args.location.sat_ordinal} - `; - if (dup.count > 0) { - logger.warn( - { - block_height: args.location.block_height, - genesis_id: args.inscription.genesis_id, - }, - `PgStore inserting duplicate blessed inscription in satoshi ${args.location.sat_ordinal}` - ); - } - } - } - const inscription = await sql<{ id: number }[]>` INSERT INTO inscriptions ${sql(args.inscription)} ON CONFLICT ON CONSTRAINT inscriptions_number_unique DO UPDATE SET @@ -674,12 +584,16 @@ export class PgStore extends BasePgStore { content_type = EXCLUDED.content_type, content_length = EXCLUDED.content_length, content = EXCLUDED.content, - fee = EXCLUDED.fee + fee = EXCLUDED.fee, + sat_ordinal = EXCLUDED.sat_ordinal, + sat_rarity = EXCLUDED.sat_rarity, + sat_coinbase_height = EXCLUDED.sat_coinbase_height RETURNING id `; inscription_id = inscription[0].id; const location = { inscription_id, + genesis_id: args.location.genesis_id, block_height: args.location.block_height, block_hash: args.location.block_hash, tx_id: args.location.tx_id, @@ -689,23 +603,18 @@ export class PgStore extends BasePgStore { prev_output: args.location.prev_output, prev_offset: args.location.prev_offset, value: args.location.value, - sat_ordinal: args.location.sat_ordinal, - sat_rarity: args.location.sat_rarity, - sat_coinbase_height: args.location.sat_coinbase_height, timestamp: sql`to_timestamp(${args.location.timestamp})`, }; await sql` INSERT INTO locations ${sql(location)} - ON CONFLICT ON CONSTRAINT locations_inscription_id_block_height_unique DO UPDATE SET + ON CONFLICT ON CONSTRAINT locations_genesis_id_block_height_unique DO UPDATE SET + inscription_id = EXCLUDED.inscription_id, block_hash = EXCLUDED.block_hash, tx_id = EXCLUDED.tx_id, address = EXCLUDED.address, output = EXCLUDED.output, "offset" = EXCLUDED.offset, value = EXCLUDED.value, - sat_ordinal = EXCLUDED.sat_ordinal, - sat_rarity = EXCLUDED.sat_rarity, - sat_coinbase_height = EXCLUDED.sat_coinbase_height, timestamp = EXCLUDED.timestamp `; const json = inscriptionContentToJson(args.inscription); @@ -724,44 +633,74 @@ export class PgStore extends BasePgStore { content = EXCLUDED.content `; } + logger.info( + `PgStore${upsert.count > 0 ? ' upsert ' : ' '}reveal #${args.inscription.number} (${ + args.location.genesis_id + }) at block ${args.location.block_height}` + ); }); return inscription_id; } - private async insertInscriptionTransfer(args: { - inscription_id: number; - location: DbLocationInsert; - }): Promise { - const location = { - inscription_id: args.inscription_id, - block_height: args.location.block_height, - block_hash: args.location.block_hash, - tx_id: args.location.tx_id, - address: args.location.address, - output: args.location.output, - offset: args.location.offset, - prev_output: args.location.prev_output, - prev_offset: args.location.prev_offset, - value: args.location.value, - sat_ordinal: args.location.sat_ordinal, - sat_rarity: args.location.sat_rarity, - sat_coinbase_height: args.location.sat_coinbase_height, - timestamp: this.sql`to_timestamp(${args.location.timestamp})`, - }; - await this.sql` - INSERT INTO locations ${this.sql(location)} - ON CONFLICT ON CONSTRAINT locations_inscription_id_block_height_unique DO UPDATE SET - block_hash = EXCLUDED.block_hash, - tx_id = EXCLUDED.tx_id, - address = EXCLUDED.address, - output = EXCLUDED.output, - "offset" = EXCLUDED.offset, - value = EXCLUDED.value, - sat_ordinal = EXCLUDED.sat_ordinal, - sat_rarity = EXCLUDED.sat_rarity, - sat_coinbase_height = EXCLUDED.sat_coinbase_height, - timestamp = EXCLUDED.timestamp - `; + private async insertLocation(args: { location: DbLocationInsert }): Promise { + await this.sqlWriteTransaction(async sql => { + // Does the inscription exist? Warn if it doesn't. + const genesis = await sql` + SELECT id FROM inscriptions WHERE genesis_id = ${args.location.genesis_id} + `; + if (genesis.count === 0) { + logger.warn( + `PgStore inserting transfer for missing inscription (${args.location.genesis_id}) at block ${args.location.block_height}` + ); + } + // Do we have the location from `prev_output`? Warn if we don't. + if (args.location.prev_output) { + const prev = await sql` + SELECT id FROM locations + WHERE genesis_id = ${args.location.genesis_id} + AND prev_output = ${args.location.prev_output} + `; + if (prev.count === 0) { + logger.warn( + `PgStore inserting transfer (${args.location.genesis_id}) superceding a missing prev_output ${args.location.prev_output} at block ${args.location.block_height}` + ); + } + } + const upsert = await sql` + SELECT id FROM locations + WHERE genesis_id = ${args.location.genesis_id} + AND block_height = ${args.location.block_height} + `; + const location = { + genesis_id: args.location.genesis_id, + block_height: args.location.block_height, + block_hash: args.location.block_hash, + tx_id: args.location.tx_id, + address: args.location.address, + output: args.location.output, + offset: args.location.offset, + prev_output: args.location.prev_output, + prev_offset: args.location.prev_offset, + value: args.location.value, + timestamp: this.sql`to_timestamp(${args.location.timestamp})`, + }; + await this.sql` + INSERT INTO locations ${this.sql(location)} + ON CONFLICT ON CONSTRAINT locations_genesis_id_block_height_unique DO UPDATE SET + block_hash = EXCLUDED.block_hash, + tx_id = EXCLUDED.tx_id, + address = EXCLUDED.address, + output = EXCLUDED.output, + "offset" = EXCLUDED.offset, + value = EXCLUDED.value, + timestamp = EXCLUDED.timestamp + `; + logger.info( + `PgStore${upsert.count > 0 ? ' upsert ' : ' '}transfer (${ + args.location.genesis_id + }) at block ${args.location.block_height}` + ); + }); } private async normalizeInscriptionCount(args: { min_block_height: number }): Promise { @@ -803,52 +742,56 @@ export class PgStore extends BasePgStore { }); } - private async rollBackInscriptionGenesis(args: { genesis_id: string }): Promise { + private async rollBackInscription(args: { + genesis_id: string; + number: number; + block_height: number; + }): Promise { // This will cascade into dependent tables. await this.sql`DELETE FROM inscriptions WHERE genesis_id = ${args.genesis_id}`; + logger.info( + `PgStore rollback reveal #${args.number} (${args.genesis_id}) at block ${args.block_height}` + ); } - private async rollBackInscriptionTransfer(args: { + private async rollBackLocation(args: { genesis_id: string; output: string; - }): Promise { - let inscription_id: number | undefined; - await this.sqlWriteTransaction(async sql => { - const inscription = await sql<{ id: number }[]>` - SELECT id FROM inscriptions WHERE genesis_id = ${args.genesis_id} - `; - if (inscription.count === 0) { - logger.warn(args, `PgStore ignoring rollback for a transfer that does not exist`); - return; - } - inscription_id = inscription[0].id; - await sql` - DELETE FROM locations - WHERE inscription_id = ${inscription_id} AND output = ${args.output} - `; - }); - return inscription_id; + block_height: number; + }): Promise { + await this.sql` + DELETE FROM locations + WHERE genesis_id = ${args.genesis_id} AND output = ${args.output} + `; + logger.info( + `PgStore rollback transfer (${args.genesis_id}) on output ${args.output} at block ${args.block_height}` + ); } - private async normalizeInscriptionLocations(args: { inscription_id: number[] }): Promise { + private async normalizeInscriptionLocations(args: { genesis_id: string[] }): Promise { await this.sqlWriteTransaction(async sql => { - for (const id of args.inscription_id) { + for (const genesis_id of args.genesis_id) { await sql` WITH i_genesis AS ( SELECT id FROM locations - WHERE inscription_id = ${id} + WHERE genesis_id = ${genesis_id} ORDER BY block_height ASC LIMIT 1 ), i_current AS ( SELECT id FROM locations - WHERE inscription_id = ${id} + WHERE genesis_id = ${genesis_id} ORDER BY block_height DESC LIMIT 1 + ), i_id AS ( + SELECT id FROM inscriptions + WHERE genesis_id = ${genesis_id} + LIMIT 1 ) UPDATE locations SET + inscription_id = (SELECT id FROM i_id), current = (CASE WHEN id = (SELECT id FROM i_current) THEN TRUE ELSE FALSE END), genesis = (CASE WHEN id = (SELECT id FROM i_genesis) THEN TRUE ELSE FALSE END) - WHERE inscription_id = ${id} + WHERE genesis_id = ${genesis_id} `; } }); diff --git a/src/pg/types.ts b/src/pg/types.ts index 18e0b1b6..967c01da 100644 --- a/src/pg/types.ts +++ b/src/pg/types.ts @@ -43,15 +43,13 @@ export type DbLocationInsert = { prev_output: string | null; prev_offset: PgNumeric | null; value: PgNumeric | null; - sat_ordinal: PgNumeric; - sat_rarity: string; - sat_coinbase_height: number; timestamp: number; }; export type DbLocation = { id: string; - inscription_id: string; + inscription_id: string | null; + genesis_id: string; block_height: string; block_hash: string; tx_id: string; @@ -61,9 +59,6 @@ export type DbLocation = { prev_output: string | null; prev_offset: string | null; value: string | null; - sat_ordinal: string; - sat_rarity: string; - sat_coinbase_height: string; timestamp: Date; genesis: boolean; current: boolean; @@ -81,9 +76,6 @@ export type DbInscriptionLocationChange = { from_output: string; from_offset: string | null; from_value: string | null; - from_sat_ordinal: string; - from_sat_rarity: string; - from_sat_coinbase_height: string; from_timestamp: Date; from_genesis: boolean; from_current: boolean; @@ -96,9 +88,6 @@ export type DbInscriptionLocationChange = { to_output: string; to_offset: string | null; to_value: string | null; - to_sat_ordinal: string; - to_sat_rarity: string; - to_sat_coinbase_height: string; to_timestamp: Date; to_genesis: boolean; to_current: boolean; @@ -107,6 +96,7 @@ export type DbInscriptionLocationChange = { export const LOCATIONS_COLUMNS = [ 'id', 'inscription_id', + 'genesis_id', 'block_height', 'block_hash', 'tx_id', @@ -114,9 +104,6 @@ export const LOCATIONS_COLUMNS = [ 'output', 'offset', 'value', - 'sat_ordinal', - 'sat_rarity', - 'sat_coinbase_height', 'timestamp', 'genesis', 'current', @@ -131,6 +118,9 @@ export type DbInscriptionInsert = { content: PgBytea; fee: PgNumeric; curse_type: string | null; + sat_ordinal: PgNumeric; + sat_rarity: string; + sat_coinbase_height: number; }; export type DbInscription = { @@ -141,6 +131,9 @@ export type DbInscription = { content_type: string; content_length: string; fee: string; + sat_ordinal: string; + sat_rarity: string; + sat_coinbase_height: string; }; export type DbInscriptionContent = { @@ -158,6 +151,9 @@ export const INSCRIPTIONS_COLUMNS = [ 'content_length', 'fee', 'curse_type', + 'sat_ordinal', + 'sat_rarity', + 'sat_coinbase_height', ]; export type DbJsonContent = { diff --git a/tests/server.test.ts b/tests/server.test.ts index f7861cc3..7b67463d 100644 --- a/tests/server.test.ts +++ b/tests/server.test.ts @@ -3,23 +3,27 @@ import { PREDICATE_UUID, startChainhookServer } from '../src/chainhook/server'; import { ENV } from '../src/env'; import { cycleMigrations } from '../src/pg/migrations'; import { PgStore } from '../src/pg/pg-store'; -import { TestChainhookPayloadBuilder } from './helpers'; +import { TestChainhookPayloadBuilder, TestFastifyServer } from './helpers'; import { ChainhookEventObserver, Payload } from '@hirosystems/chainhook-client'; +import { buildApiServer } from '../src/api/init'; describe('EventServer', () => { let db: PgStore; let server: ChainhookEventObserver; + let fastify: TestFastifyServer; beforeEach(async () => { db = await PgStore.connect({ skipMigrations: true }); await cycleMigrations(); ENV.CHAINHOOK_AUTO_PREDICATE_REGISTRATION = false; server = await startChainhookServer({ db }); + fastify = await buildApiServer({ db }); }); afterEach(async () => { - await db.close(); await server.close(); + await fastify.close(); + await db.close(); }); describe('parser', () => { @@ -311,6 +315,122 @@ describe('EventServer', () => { expect(c2[0].count).toBe(1); }); + test('saves transfer without genesis and fills the gap later', async () => { + // Insert transfers with no genesis + await db.updateInscriptions( + new TestChainhookPayloadBuilder() + .apply() + .block({ + height: 775620, + hash: '0x00000000000000000002a90330a99f67e3f01eb2ce070b45930581e82fb7a91d', + timestamp: 1676913207, + }) + .transaction({ + hash: '0x38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', + }) + .inscriptionTransferred({ + inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', + updated_address: 'bc1qcf3dgqgvylmd5ayl4njm4ephqfdazy93ssu28j', + satpoint_pre_transfer: + '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0:0', + satpoint_post_transfer: + '9e2414153b1893f799477f7e1a00a52fafc235de72fd215cb3321f253c4464ac:0:0', + post_transfer_output_value: 9000, + }) + .build() + ); + await db.updateInscriptions( + new TestChainhookPayloadBuilder() + .apply() + .block({ + height: 775621, + hash: '00000000000000000003dd4738355bedb73796de9b1099e59ff7adc235e967a6', + timestamp: 1676913207, + }) + .transaction({ + hash: '2fa1640d61f04a699833f0f6a884f543c835fc60f0fd4da8627ebb857acdce84', + }) + .inscriptionTransferred({ + inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', + updated_address: 'bc1qcf3dgqgvylmd5ayl4njm4ephqfdazy93ssu28j', + satpoint_pre_transfer: + '9e2414153b1893f799477f7e1a00a52fafc235de72fd215cb3321f253c4464ac:0:0', + satpoint_post_transfer: + '2fa1640d61f04a699833f0f6a884f543c835fc60f0fd4da8627ebb857acdce84:0:0', + post_transfer_output_value: 8000, + }) + .build() + ); + // Locations should exist with null FKs + const results1 = await db.sql` + SELECT * FROM locations + WHERE genesis_id = '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0' + `; + expect(results1.count).toBe(2); + expect(results1[0].inscription_id).toBeNull(); + expect(results1[1].inscription_id).toBeNull(); + const api1 = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', + }); + expect(api1.statusCode).toBe(404); + const api2 = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0/transfers', + }); + expect(api2.json().total).toBe(0); + + // Insert genesis and make sure locations are normalized. + await db.updateInscriptions( + new TestChainhookPayloadBuilder() + .apply() + .block({ + height: 775618, + hash: '0x00000000000000000002a90330a99f67e3f01eb2ce070b45930581e82fb7a91d', + timestamp: 1676913207, + }) + .transaction({ + hash: '0x38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', + }) + .inscriptionRevealed({ + content_bytes: '0x48656C6C6F', + content_type: 'image/png', + content_length: 5, + inscription_number: 7, + inscription_fee: 2805, + inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', + inscription_output_value: 10000, + inscriber_address: 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + ordinal_number: 5, + ordinal_block_height: 0, + ordinal_offset: 0, + satpoint_post_inscription: + '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0:0', + }) + .build() + ); + // Locations should exist with all FKs filled in + const results2 = await db.sql` + SELECT * FROM locations + WHERE genesis_id = '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0' + `; + expect(results2.count).toBe(3); + expect(results2[0].inscription_id).not.toBeNull(); + expect(results2[1].inscription_id).not.toBeNull(); + expect(results2[2].inscription_id).not.toBeNull(); + const api3 = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', + }); + expect(api3.statusCode).toBe(200); + expect(api3.json().genesis_block_height).toBe(775618); + const api4 = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0/transfers', + }); + expect(api4.json().total).toBe(3); + }); + test('saves p/op json content', async () => { const reveal = { block_identifier: {