From 5422156e9919f0c5870c9571ea9f591852c98b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Fri, 26 Apr 2024 10:33:35 -0600 Subject: [PATCH] feat!: support reinscription transfers (#348) * chore: progress * feat: new apply * fix: minted supply * fix: track tx counts * feat: rollbacks * fix: transfer rollbacks * fix: activity addresses * fix: multiple transfer test * fix: holders * fix: operation indexes * chore: draft * fix: style * chore: optimize migrations * fix: initial inserts * fix: get inscription endpoint * fix: rollback * fix: ordhook tests passing * chore: logging * fix: recursion refs * fix: current location comparison * fix: transfers table * test: sat reinscription transfers * fix: start applying counts * fix: tests * fix: all api tests passing * test: transfers per block reinscription * test: address transfer counts * fix: brc20 tests * chore: refactor block caches * chore: clean unused exports --- .github/workflows/ci.yml | 3 + migrations/1676395230925_satoshis.ts | 22 + migrations/1676395230930_inscriptions.ts | 50 +- migrations/1677284495299_locations.ts | 41 +- ....ts => 1677284495500_current-locations.ts} | 20 +- .../1677284495501_inscription-transfers.ts | 52 ++ ...ip-table.ts => 1677284495900_chain-tip.ts} | 14 +- .../1677284495992_inscription-recursions.ts | 25 + ...ck.ts => 1677284495995_counts-by-block.ts} | 3 +- migrations/1677360299810_chain-tip.ts | 14 - ...s => 1683047918926_counts-by-mime-type.ts} | 16 +- migrations/1683047918926_mime-type-counts.ts | 13 - ... => 1683061444855_counts-by-sat-rarity.ts} | 12 +- migrations/1683061444855_sat-rarity-counts.ts | 17 - ...ent.ts => 1683130423352_counts-by-type.ts} | 11 +- migrations/1688925112931_genesis-locations.ts | 36 - migrations/1689264599745_address-counts.ts | 13 - ....ts => 1689264599745_counts-by-address.ts} | 12 +- ...689264599745_counts-by-genesis-address.ts} | 17 +- .../1690229956705_inscription-recursions.ts | 46 - .../1690476164909_count-views-to-tables.ts | 149 --- ...0832271103_location-pointer-constraints.ts | 55 -- migrations/1692980393413_locations-unique.ts | 34 - ...572099_locations-remove-duplicate-index.ts | 12 - .../1693235147508_recursion-backfills.ts | 51 -- .../1695655140203_counts-by-recursive.ts | 11 +- .../1698897577725_locations-location-index.ts | 14 - ...63472553_locations-block-height-indexes.ts | 22 - .../1708471015438_remove-unused-indexes.ts | 18 - package-lock.json | 35 + package.json | 2 + src/api/routes/stats.ts | 2 +- src/api/schemas.ts | 12 +- src/api/util/cache.ts | 7 +- src/api/util/helpers.ts | 2 +- src/ordhook/server.ts | 3 +- src/pg/block-cache.ts | 193 ++++ src/pg/brc20/brc20-block-cache.ts | 228 +++++ src/pg/brc20/brc20-pg-store.ts | 216 ++--- src/pg/brc20/helpers.ts | 89 -- src/pg/brc20/types.ts | 64 +- src/pg/counts/counts-pg-store.ts | 249 +++-- src/pg/helpers.ts | 166 +--- src/pg/pg-store.ts | 853 ++++++++---------- src/pg/types.ts | 136 +-- tests/api/inscriptions.test.ts | 17 +- tests/api/sats.test.ts | 60 +- tests/ordhook/server.test.ts | 26 + 48 files changed, 1355 insertions(+), 1808 deletions(-) create mode 100644 migrations/1676395230925_satoshis.ts rename migrations/{1689006001522_current-locations.ts => 1677284495500_current-locations.ts} (56%) create mode 100644 migrations/1677284495501_inscription-transfers.ts rename migrations/{1701486147464_chain-tip-table.ts => 1677284495900_chain-tip.ts} (59%) create mode 100644 migrations/1677284495992_inscription-recursions.ts rename migrations/{1687785552000_inscriptions-per-block.ts => 1677284495995_counts-by-block.ts} (87%) delete mode 100644 migrations/1677360299810_chain-tip.ts rename migrations/{1704341578275_jubilee-numbers.ts => 1683047918926_counts-by-mime-type.ts} (58%) delete mode 100644 migrations/1683047918926_mime-type-counts.ts rename migrations/{1693234845450_locations-null-inscription-id-index.ts => 1683061444855_counts-by-sat-rarity.ts} (56%) delete mode 100644 migrations/1683061444855_sat-rarity-counts.ts rename migrations/{1707770109739_metadata-parent.ts => 1683130423352_counts-by-type.ts} (66%) delete mode 100644 migrations/1688925112931_genesis-locations.ts delete mode 100644 migrations/1689264599745_address-counts.ts rename migrations/{1698856424356_locations-transfer-type.ts => 1689264599745_counts-by-address.ts} (64%) rename migrations/{1683130423352_inscription-count.ts => 1689264599745_counts-by-genesis-address.ts} (55%) delete mode 100644 migrations/1690229956705_inscription-recursions.ts delete mode 100644 migrations/1690476164909_count-views-to-tables.ts delete mode 100644 migrations/1690832271103_location-pointer-constraints.ts delete mode 100644 migrations/1692980393413_locations-unique.ts delete mode 100644 migrations/1693234572099_locations-remove-duplicate-index.ts delete mode 100644 migrations/1693235147508_recursion-backfills.ts delete mode 100644 migrations/1698897577725_locations-location-index.ts delete mode 100644 migrations/1705363472553_locations-block-height-indexes.ts delete mode 100644 migrations/1708471015438_remove-unused-indexes.ts create mode 100644 src/pg/block-cache.ts create mode 100644 src/pg/brc20/brc20-block-cache.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72828daa..d8d328a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,9 @@ jobs: - name: Lint Prettier run: npm run lint:prettier + - name: Lint Unused Exports + run: npm run lint:unused-exports + test: strategy: fail-fast: false diff --git a/migrations/1676395230925_satoshis.ts b/migrations/1676395230925_satoshis.ts new file mode 100644 index 00000000..62c1d0b7 --- /dev/null +++ b/migrations/1676395230925_satoshis.ts @@ -0,0 +1,22 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export function up(pgm: MigrationBuilder): void { + pgm.createTable('satoshis', { + ordinal_number: { + type: 'numeric', + primaryKey: true, + }, + rarity: { + type: 'text', + notNull: true, + }, + coinbase_height: { + type: 'bigint', + notNull: true, + }, + }); + pgm.createIndex('satoshis', ['rarity']); +} diff --git a/migrations/1676395230930_inscriptions.ts b/migrations/1676395230930_inscriptions.ts index 8b4bd09c..3205872b 100644 --- a/migrations/1676395230930_inscriptions.ts +++ b/migrations/1676395230930_inscriptions.ts @@ -5,30 +5,33 @@ export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { pgm.createTable('inscriptions', { - id: { - type: 'bigserial', - primaryKey: true, - }, genesis_id: { type: 'text', + primaryKey: true, + }, + ordinal_number: { + type: 'numeric', notNull: true, }, number: { type: 'bigint', notNull: true, }, - sat_ordinal: { - type: 'numeric', + classic_number: { + type: 'bigint', notNull: true, }, - sat_rarity: { - type: 'text', + block_height: { + type: 'bigint', notNull: true, }, - sat_coinbase_height: { + tx_index: { type: 'bigint', notNull: true, }, + address: { + type: 'text', + }, mime_type: { type: 'text', notNull: true, @@ -52,6 +55,20 @@ export function up(pgm: MigrationBuilder): void { curse_type: { type: 'text', }, + recursive: { + type: 'boolean', + default: false, + }, + metadata: { + type: 'text', + }, + parent: { + type: 'text', + }, + timestamp: { + type: 'timestamptz', + notNull: true, + }, updated_at: { type: 'timestamptz', default: pgm.func('(NOW())'), @@ -59,10 +76,17 @@ export function up(pgm: MigrationBuilder): void { }, }); pgm.createConstraint('inscriptions', 'inscriptions_number_unique', 'UNIQUE(number)'); - pgm.createIndex('inscriptions', ['genesis_id']); + pgm.createConstraint( + 'inscriptions', + 'inscriptions_ordinal_number_fk', + 'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE' + ); pgm.createIndex('inscriptions', ['mime_type']); - pgm.createIndex('inscriptions', ['sat_ordinal']); - pgm.createIndex('inscriptions', ['sat_rarity']); - pgm.createIndex('inscriptions', ['sat_coinbase_height']); + pgm.createIndex('inscriptions', ['recursive']); + pgm.createIndex('inscriptions', [ + { name: 'block_height', sort: 'DESC' }, + { name: 'tx_index', sort: 'DESC' }, + ]); + pgm.createIndex('inscriptions', ['address']); pgm.createIndex('inscriptions', [{ name: 'updated_at', sort: 'DESC' }]); } diff --git a/migrations/1677284495299_locations.ts b/migrations/1677284495299_locations.ts index b9cc76b1..30894492 100644 --- a/migrations/1677284495299_locations.ts +++ b/migrations/1677284495299_locations.ts @@ -4,32 +4,26 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { + pgm.createType('transfer_type', ['transferred', 'spent_in_fees', 'burnt']); pgm.createTable('locations', { - id: { - type: 'bigserial', - primaryKey: true, - }, - inscription_id: { - type: 'bigint', - }, - genesis_id: { - type: 'text', + ordinal_number: { + type: 'numeric', notNull: true, }, block_height: { type: 'bigint', notNull: true, }, - block_hash: { - type: 'text', + tx_index: { + type: 'bigint', notNull: true, }, tx_id: { type: 'text', notNull: true, }, - tx_index: { - type: 'bigint', + block_hash: { + type: 'text', notNull: true, }, address: { @@ -51,26 +45,27 @@ export function up(pgm: MigrationBuilder): void { value: { type: 'numeric', }, + transfer_type: { + type: 'transfer_type', + notNull: true, + }, timestamp: { type: 'timestamptz', notNull: true, }, }); + pgm.createConstraint('locations', 'locations_pkey', { + primaryKey: ['ordinal_number', 'block_height', 'tx_index'], + }); pgm.createConstraint( 'locations', - 'locations_inscription_id_fk', - 'FOREIGN KEY(inscription_id) REFERENCES inscriptions(id) ON DELETE CASCADE' + 'locations_ordinal_number_fk', + 'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE' ); - pgm.createConstraint('locations', 'locations_output_offset_unique', 'UNIQUE(output, "offset")'); - pgm.createIndex('locations', ['inscription_id']); + pgm.createIndex('locations', ['output', 'offset']); + pgm.createIndex('locations', ['timestamp']); pgm.createIndex('locations', [ - 'genesis_id', { name: 'block_height', sort: 'DESC' }, { name: 'tx_index', sort: 'DESC' }, ]); - pgm.createIndex('locations', ['block_height']); - pgm.createIndex('locations', ['block_hash']); - pgm.createIndex('locations', ['address']); - pgm.createIndex('locations', ['timestamp']); - pgm.createIndex('locations', ['prev_output']); } diff --git a/migrations/1689006001522_current-locations.ts b/migrations/1677284495500_current-locations.ts similarity index 56% rename from migrations/1689006001522_current-locations.ts rename to migrations/1677284495500_current-locations.ts index 3a469202..51f4b8a3 100644 --- a/migrations/1689006001522_current-locations.ts +++ b/migrations/1677284495500_current-locations.ts @@ -5,12 +5,8 @@ export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { pgm.createTable('current_locations', { - inscription_id: { - type: 'bigint', - notNull: true, - }, - location_id: { - type: 'bigint', + ordinal_number: { + type: 'numeric', notNull: true, }, block_height: { @@ -27,10 +23,14 @@ export function up(pgm: MigrationBuilder): void { }); pgm.createConstraint( 'current_locations', - 'current_locations_inscription_id_unique', - 'UNIQUE(inscription_id)' + 'current_locations_locations_fk', + 'FOREIGN KEY(ordinal_number, block_height, tx_index) REFERENCES locations(ordinal_number, block_height, tx_index) ON DELETE CASCADE' + ); + pgm.createConstraint( + 'locations', + 'locations_satoshis_fk', + 'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE' ); - pgm.createIndex('current_locations', ['location_id']); - pgm.createIndex('current_locations', ['block_height']); + pgm.createIndex('current_locations', ['ordinal_number'], { unique: true }); pgm.createIndex('current_locations', ['address']); } diff --git a/migrations/1677284495501_inscription-transfers.ts b/migrations/1677284495501_inscription-transfers.ts new file mode 100644 index 00000000..90b72717 --- /dev/null +++ b/migrations/1677284495501_inscription-transfers.ts @@ -0,0 +1,52 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export function up(pgm: MigrationBuilder): void { + pgm.createTable('inscription_transfers', { + genesis_id: { + type: 'text', + notNull: true, + }, + number: { + type: 'bigint', + notNull: true, + }, + ordinal_number: { + type: 'numeric', + notNull: true, + }, + block_height: { + type: 'bigint', + notNull: true, + }, + tx_index: { + type: 'bigint', + notNull: true, + }, + block_hash: { + type: 'text', + notNull: true, + }, + block_transfer_index: { + type: 'int', + notNull: true, + }, + }); + pgm.createConstraint('inscription_transfers', 'inscription_transfers_pkey', { + primaryKey: ['block_height', 'block_transfer_index'], + }); + pgm.createConstraint( + 'inscription_transfers', + 'inscription_transfers_locations_fk', + 'FOREIGN KEY(ordinal_number, block_height, tx_index) REFERENCES locations(ordinal_number, block_height, tx_index) ON DELETE CASCADE' + ); + pgm.createConstraint( + 'inscription_transfers', + 'inscription_transfers_satoshis_fk', + 'FOREIGN KEY(ordinal_number) REFERENCES satoshis(ordinal_number) ON DELETE CASCADE' + ); + pgm.createIndex('inscription_transfers', ['genesis_id']); + pgm.createIndex('inscription_transfers', ['number']); +} diff --git a/migrations/1701486147464_chain-tip-table.ts b/migrations/1677284495900_chain-tip.ts similarity index 59% rename from migrations/1701486147464_chain-tip-table.ts rename to migrations/1677284495900_chain-tip.ts index 1f9b30b2..2b897d40 100644 --- a/migrations/1701486147464_chain-tip-table.ts +++ b/migrations/1677284495900_chain-tip.ts @@ -4,7 +4,6 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.dropMaterializedView('chain_tip'); pgm.createTable('chain_tip', { id: { type: 'bool', @@ -19,20 +18,9 @@ export function up(pgm: MigrationBuilder): void { }, }); pgm.addConstraint('chain_tip', 'chain_tip_one_row', 'CHECK(id)'); - pgm.sql(` - INSERT INTO chain_tip (block_height) ( - SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations - ) - `); + pgm.sql(`INSERT INTO chain_tip DEFAULT VALUES`); } export function down(pgm: MigrationBuilder): void { pgm.dropTable('chain_tip'); - pgm.createMaterializedView( - 'chain_tip', - { data: true }, - // Set block height 767430 (inscription #0 genesis) as default. - `SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations` - ); - pgm.createIndex('chain_tip', ['block_height'], { unique: true }); } diff --git a/migrations/1677284495992_inscription-recursions.ts b/migrations/1677284495992_inscription-recursions.ts new file mode 100644 index 00000000..d75fb405 --- /dev/null +++ b/migrations/1677284495992_inscription-recursions.ts @@ -0,0 +1,25 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; + +export const shorthands: ColumnDefinitions | undefined = undefined; + +export function up(pgm: MigrationBuilder): void { + pgm.createTable('inscription_recursions', { + genesis_id: { + type: 'text', + notNull: true, + }, + ref_genesis_id: { + type: 'text', + notNull: true, + }, + }); + pgm.createConstraint('inscription_recursions', 'inscription_recursions_pkey', { + primaryKey: ['genesis_id', 'ref_genesis_id'], + }); + pgm.createConstraint( + 'inscription_recursions', + 'inscription_recursions_genesis_id_fk', + 'FOREIGN KEY(genesis_id) REFERENCES inscriptions(genesis_id) ON DELETE CASCADE' + ); +} diff --git a/migrations/1687785552000_inscriptions-per-block.ts b/migrations/1677284495995_counts-by-block.ts similarity index 87% rename from migrations/1687785552000_inscriptions-per-block.ts rename to migrations/1677284495995_counts-by-block.ts index 7aef97a8..2c33335c 100644 --- a/migrations/1687785552000_inscriptions-per-block.ts +++ b/migrations/1677284495995_counts-by-block.ts @@ -4,7 +4,7 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.createTable('inscriptions_per_block', { + pgm.createTable('counts_by_block', { block_height: { type: 'bigint', primaryKey: true, @@ -26,4 +26,5 @@ export function up(pgm: MigrationBuilder): void { notNull: true, }, }); + pgm.createIndex('counts_by_block', ['block_hash']); } diff --git a/migrations/1677360299810_chain-tip.ts b/migrations/1677360299810_chain-tip.ts deleted file mode 100644 index 7dbab3d3..00000000 --- a/migrations/1677360299810_chain-tip.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createMaterializedView( - 'chain_tip', - { data: true }, - // Set block height 767430 (inscription #0 genesis) as default. - `SELECT GREATEST(MAX(block_height), 767430) AS block_height FROM locations` - ); - pgm.createIndex('chain_tip', ['block_height'], { unique: true }); -} diff --git a/migrations/1704341578275_jubilee-numbers.ts b/migrations/1683047918926_counts-by-mime-type.ts similarity index 58% rename from migrations/1704341578275_jubilee-numbers.ts rename to migrations/1683047918926_counts-by-mime-type.ts index bd92ae06..8b0de2ba 100644 --- a/migrations/1704341578275_jubilee-numbers.ts +++ b/migrations/1683047918926_counts-by-mime-type.ts @@ -4,13 +4,15 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.addColumn('inscriptions', { - classic_number: { - type: 'bigint', + pgm.createTable('counts_by_mime_type', { + mime_type: { + type: 'text', + primaryKey: true, + }, + count: { + type: 'int', + notNull: true, + default: 0, }, }); } - -export function down(pgm: MigrationBuilder): void { - pgm.dropColumn('inscriptions', 'classic_number'); -} diff --git a/migrations/1683047918926_mime-type-counts.ts b/migrations/1683047918926_mime-type-counts.ts deleted file mode 100644 index be4d52d1..00000000 --- a/migrations/1683047918926_mime-type-counts.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createMaterializedView( - 'mime_type_counts', - { data: true }, - `SELECT mime_type, COUNT(*) AS count FROM inscriptions GROUP BY mime_type` - ); - pgm.createIndex('mime_type_counts', ['mime_type'], { unique: true }); -} diff --git a/migrations/1693234845450_locations-null-inscription-id-index.ts b/migrations/1683061444855_counts-by-sat-rarity.ts similarity index 56% rename from migrations/1693234845450_locations-null-inscription-id-index.ts rename to migrations/1683061444855_counts-by-sat-rarity.ts index c522d1c3..5fc074ce 100644 --- a/migrations/1693234845450_locations-null-inscription-id-index.ts +++ b/migrations/1683061444855_counts-by-sat-rarity.ts @@ -4,5 +4,15 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.createIndex('locations', ['inscription_id'], { where: 'inscription_id IS NULL' }); + pgm.createTable('counts_by_sat_rarity', { + sat_rarity: { + type: 'text', + primaryKey: true, + }, + count: { + type: 'int', + notNull: true, + default: 0, + }, + }); } diff --git a/migrations/1683061444855_sat-rarity-counts.ts b/migrations/1683061444855_sat-rarity-counts.ts deleted file mode 100644 index 921d59d0..00000000 --- a/migrations/1683061444855_sat-rarity-counts.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createMaterializedView( - 'sat_rarity_counts', - { data: true }, - ` - SELECT sat_rarity, COUNT(*) AS count - FROM inscriptions AS i - GROUP BY sat_rarity - ` - ); - pgm.createIndex('sat_rarity_counts', ['sat_rarity'], { unique: true }); -} diff --git a/migrations/1707770109739_metadata-parent.ts b/migrations/1683130423352_counts-by-type.ts similarity index 66% rename from migrations/1707770109739_metadata-parent.ts rename to migrations/1683130423352_counts-by-type.ts index 0c33c976..f7de9393 100644 --- a/migrations/1707770109739_metadata-parent.ts +++ b/migrations/1683130423352_counts-by-type.ts @@ -4,12 +4,15 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.addColumns('inscriptions', { - metadata: { + pgm.createTable('counts_by_type', { + type: { type: 'text', + primaryKey: true, }, - parent: { - type: 'text', + count: { + type: 'int', + notNull: true, + default: 0, }, }); } diff --git a/migrations/1688925112931_genesis-locations.ts b/migrations/1688925112931_genesis-locations.ts deleted file mode 100644 index 543c61e1..00000000 --- a/migrations/1688925112931_genesis-locations.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createTable('genesis_locations', { - inscription_id: { - type: 'bigint', - notNull: true, - }, - location_id: { - type: 'bigint', - notNull: true, - }, - block_height: { - type: 'bigint', - notNull: true, - }, - tx_index: { - type: 'bigint', - notNull: true, - }, - address: { - type: 'text', - }, - }); - pgm.createConstraint( - 'genesis_locations', - 'genesis_locations_inscription_id_unique', - 'UNIQUE(inscription_id)' - ); - pgm.createIndex('genesis_locations', ['location_id']); - pgm.createIndex('genesis_locations', ['block_height']); - pgm.createIndex('genesis_locations', ['address']); -} diff --git a/migrations/1689264599745_address-counts.ts b/migrations/1689264599745_address-counts.ts deleted file mode 100644 index 4a21827e..00000000 --- a/migrations/1689264599745_address-counts.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createMaterializedView( - 'address_counts', - { data: true }, - `SELECT address, COUNT(*) AS count FROM current_locations GROUP BY address` - ); - pgm.createIndex('address_counts', ['address'], { unique: true }); -} diff --git a/migrations/1698856424356_locations-transfer-type.ts b/migrations/1689264599745_counts-by-address.ts similarity index 64% rename from migrations/1698856424356_locations-transfer-type.ts rename to migrations/1689264599745_counts-by-address.ts index c3ba335f..1e0bd9e3 100644 --- a/migrations/1698856424356_locations-transfer-type.ts +++ b/migrations/1689264599745_counts-by-address.ts @@ -4,11 +4,15 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.createType('transfer_type', ['transferred', 'spent_in_fees', 'burnt']); - pgm.addColumn('locations', { - transfer_type: { - type: 'transfer_type', + pgm.createTable('counts_by_address', { + address: { + type: 'text', + primaryKey: true, + }, + count: { + type: 'int', notNull: true, + default: 0, }, }); } diff --git a/migrations/1683130423352_inscription-count.ts b/migrations/1689264599745_counts-by-genesis-address.ts similarity index 55% rename from migrations/1683130423352_inscription-count.ts rename to migrations/1689264599745_counts-by-genesis-address.ts index 30c5b0fa..95039cdf 100644 --- a/migrations/1683130423352_inscription-count.ts +++ b/migrations/1689264599745_counts-by-genesis-address.ts @@ -4,10 +4,15 @@ import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; export const shorthands: ColumnDefinitions | undefined = undefined; export function up(pgm: MigrationBuilder): void { - pgm.createMaterializedView( - 'inscription_count', - { data: true }, - `SELECT COUNT(*) AS count FROM inscriptions` - ); - pgm.createIndex('inscription_count', ['count'], { unique: true }); + pgm.createTable('counts_by_genesis_address', { + address: { + type: 'text', + primaryKey: true, + }, + count: { + type: 'int', + notNull: true, + default: 0, + }, + }); } diff --git a/migrations/1690229956705_inscription-recursions.ts b/migrations/1690229956705_inscription-recursions.ts deleted file mode 100644 index 83f97e65..00000000 --- a/migrations/1690229956705_inscription-recursions.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.createTable('inscription_recursions', { - id: { - type: 'bigserial', - primaryKey: true, - }, - inscription_id: { - type: 'bigint', - notNull: true, - }, - ref_inscription_id: { - type: 'bigint', - notNull: true, - }, - }); - pgm.createConstraint( - 'inscription_recursions', - 'locations_inscription_id_fk', - 'FOREIGN KEY(inscription_id) REFERENCES inscriptions(id) ON DELETE CASCADE' - ); - pgm.createConstraint( - 'inscription_recursions', - 'locations_ref_inscription_id_fk', - 'FOREIGN KEY(ref_inscription_id) REFERENCES inscriptions(id) ON DELETE CASCADE' - ); - pgm.createConstraint( - 'inscription_recursions', - 'inscriptions_inscription_id_ref_inscription_id_unique', - 'UNIQUE(inscription_id, ref_inscription_id)' - ); - pgm.createIndex('inscription_recursions', ['ref_inscription_id']); - - // Add columns to `inscriptions` table. - pgm.addColumn('inscriptions', { - recursive: { - type: 'boolean', - default: false, - }, - }); - pgm.createIndex('inscriptions', ['recursive'], { where: 'recursive = TRUE' }); -} diff --git a/migrations/1690476164909_count-views-to-tables.ts b/migrations/1690476164909_count-views-to-tables.ts deleted file mode 100644 index b6a167ca..00000000 --- a/migrations/1690476164909_count-views-to-tables.ts +++ /dev/null @@ -1,149 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropMaterializedView('mime_type_counts'); - pgm.createTable('counts_by_mime_type', { - mime_type: { - type: 'text', - notNull: true, - primaryKey: true, - }, - count: { - type: 'bigint', - notNull: true, - default: 1, - }, - }); - pgm.sql(` - INSERT INTO counts_by_mime_type ( - SELECT mime_type, COUNT(*) AS count FROM inscriptions GROUP BY mime_type - ) - `); - - pgm.dropMaterializedView('sat_rarity_counts'); - pgm.createTable('counts_by_sat_rarity', { - sat_rarity: { - type: 'text', - notNull: true, - primaryKey: true, - }, - count: { - type: 'bigint', - notNull: true, - default: 1, - }, - }); - pgm.sql(` - INSERT INTO counts_by_sat_rarity ( - SELECT sat_rarity, COUNT(*) AS count FROM inscriptions GROUP BY sat_rarity - ) - `); - - pgm.dropMaterializedView('address_counts'); - pgm.createTable('counts_by_address', { - address: { - type: 'text', - notNull: true, - primaryKey: true, - }, - count: { - type: 'bigint', - notNull: true, - default: 1, - }, - }); - pgm.sql(` - INSERT INTO counts_by_address ( - SELECT address, COUNT(*) AS count FROM current_locations GROUP BY address - ) - `); - - pgm.createTable('counts_by_genesis_address', { - address: { - type: 'text', - notNull: true, - primaryKey: true, - }, - count: { - type: 'bigint', - notNull: true, - default: 1, - }, - }); - pgm.sql(` - INSERT INTO counts_by_genesis_address ( - SELECT address, COUNT(*) AS count FROM genesis_locations GROUP BY address - ) - `); - - pgm.dropMaterializedView('inscription_count'); - pgm.createTable('counts_by_type', { - type: { - type: 'text', - notNull: true, - primaryKey: true, - }, - count: { - type: 'bigint', - notNull: true, - default: 1, - }, - }); - pgm.sql(` - INSERT INTO counts_by_type ( - SELECT 'blessed' AS type, COUNT(*) AS count FROM inscriptions WHERE number >= 0 - ) - `); - pgm.sql(` - INSERT INTO counts_by_type ( - SELECT 'cursed' AS type, COUNT(*) AS count FROM inscriptions WHERE number < 0 - ) - `); - - pgm.createIndex('inscriptions_per_block', ['block_hash']); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropTable('counts_by_mime_type'); - pgm.createMaterializedView( - 'mime_type_counts', - { data: true }, - `SELECT mime_type, COUNT(*) AS count FROM inscriptions GROUP BY mime_type` - ); - pgm.createIndex('mime_type_counts', ['mime_type'], { unique: true }); - - pgm.dropTable('counts_by_sat_rarity'); - pgm.createMaterializedView( - 'sat_rarity_counts', - { data: true }, - ` - SELECT sat_rarity, COUNT(*) AS count - FROM inscriptions AS i - GROUP BY sat_rarity - ` - ); - pgm.createIndex('sat_rarity_counts', ['sat_rarity'], { unique: true }); - - pgm.dropTable('counts_by_address'); - pgm.createMaterializedView( - 'address_counts', - { data: true }, - `SELECT address, COUNT(*) AS count FROM current_locations GROUP BY address` - ); - pgm.createIndex('address_counts', ['address'], { unique: true }); - - pgm.dropTable('counts_by_type'); - pgm.createMaterializedView( - 'inscription_count', - { data: true }, - `SELECT COUNT(*) AS count FROM inscriptions` - ); - pgm.createIndex('inscription_count', ['count'], { unique: true }); - - pgm.dropIndex('inscriptions_per_block', ['block_hash']); - - pgm.dropTable('counts_by_genesis_address'); -} diff --git a/migrations/1690832271103_location-pointer-constraints.ts b/migrations/1690832271103_location-pointer-constraints.ts deleted file mode 100644 index 07184b77..00000000 --- a/migrations/1690832271103_location-pointer-constraints.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropConstraint('genesis_locations', 'genesis_locations_inscription_id_unique'); - pgm.createConstraint('genesis_locations', 'genesis_locations_inscription_id_pk', { - primaryKey: 'inscription_id', - }); - pgm.createConstraint( - 'genesis_locations', - 'genesis_locations_inscription_id_fk', - 'FOREIGN KEY(inscription_id) REFERENCES inscriptions(id) ON DELETE CASCADE' - ); - pgm.createConstraint( - 'genesis_locations', - 'genesis_locations_location_id_fk', - 'FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE' - ); - - pgm.dropConstraint('current_locations', 'current_locations_inscription_id_unique'); - pgm.createConstraint('current_locations', 'current_locations_inscription_id_pk', { - primaryKey: 'inscription_id', - }); - pgm.createConstraint( - 'current_locations', - 'current_locations_inscription_id_fk', - 'FOREIGN KEY(inscription_id) REFERENCES inscriptions(id) ON DELETE CASCADE' - ); - pgm.createConstraint( - 'current_locations', - 'current_locations_location_id_fk', - 'FOREIGN KEY(location_id) REFERENCES locations(id) ON DELETE CASCADE' - ); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropConstraint('genesis_locations', 'genesis_locations_inscription_id_pk'); - pgm.dropConstraint('genesis_locations', 'genesis_locations_inscription_id_fk'); - pgm.dropConstraint('genesis_locations', 'genesis_locations_location_id_fk'); - pgm.createConstraint( - 'genesis_locations', - 'genesis_locations_inscription_id_unique', - 'UNIQUE(inscription_id)' - ); - pgm.dropConstraint('current_locations', 'current_locations_inscription_id_pk'); - pgm.dropConstraint('current_locations', 'current_locations_inscription_id_fk'); - pgm.dropConstraint('current_locations', 'current_locations_location_id_fk'); - pgm.createConstraint( - 'current_locations', - 'current_locations_inscription_id_unique', - 'UNIQUE(inscription_id)' - ); -} diff --git a/migrations/1692980393413_locations-unique.ts b/migrations/1692980393413_locations-unique.ts deleted file mode 100644 index 56491527..00000000 --- a/migrations/1692980393413_locations-unique.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropConstraint('locations', 'locations_output_offset_unique'); - pgm.createIndex('locations', ['output', 'offset']); - pgm.createConstraint( - 'locations', - 'locations_inscription_id_block_height_tx_index_unique', - 'UNIQUE(inscription_id, block_height, tx_index)' - ); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropConstraint('locations', 'locations_inscription_id_block_height_tx_index_unique'); - pgm.dropIndex('locations', ['output', 'offset']); - // Modify any repeated offsets slightly so we can re-add the unique constraint. This is mostly for - // unit testing purposes. - pgm.sql(` - WITH duplicates AS ( - SELECT - id, output, "offset", ROW_NUMBER() OVER (PARTITION BY output, "offset" ORDER BY id) as rn - FROM locations - ) - UPDATE locations - SET "offset" = duplicates."offset" + rn - 1 - FROM duplicates - WHERE locations.id = duplicates.id - AND rn > 1 - `); - pgm.createConstraint('locations', 'locations_output_offset_unique', 'UNIQUE(output, "offset")'); -} diff --git a/migrations/1693234572099_locations-remove-duplicate-index.ts b/migrations/1693234572099_locations-remove-duplicate-index.ts deleted file mode 100644 index e8103544..00000000 --- a/migrations/1693234572099_locations-remove-duplicate-index.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropIndex('locations', ['inscription_id'], { ifExists: true }); -} - -export function down(pgm: MigrationBuilder): void { - pgm.createIndex('locations', ['inscription_id'], { ifNotExists: true }); -} diff --git a/migrations/1693235147508_recursion-backfills.ts b/migrations/1693235147508_recursion-backfills.ts deleted file mode 100644 index d4fea843..00000000 --- a/migrations/1693235147508_recursion-backfills.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.addColumn('inscription_recursions', { - ref_inscription_genesis_id: { - type: 'text', - }, - }); - pgm.sql(` - UPDATE inscription_recursions AS ir - SET ref_inscription_genesis_id = ( - SELECT genesis_id FROM inscriptions WHERE id = ir.ref_inscription_id - ) - `); - pgm.alterColumn('inscription_recursions', 'ref_inscription_genesis_id', { notNull: true }); - pgm.alterColumn('inscription_recursions', 'ref_inscription_id', { allowNull: true }); - - pgm.createIndex('inscription_recursions', ['ref_inscription_genesis_id']); - pgm.createIndex('inscription_recursions', ['ref_inscription_id'], { - where: 'ref_inscription_id IS NULL', - name: 'inscription_recursions_ref_inscription_id_null_index', - }); - pgm.dropConstraint( - 'inscription_recursions', - 'inscriptions_inscription_id_ref_inscription_id_unique' - ); - pgm.createConstraint( - 'inscription_recursions', - 'inscription_recursions_unique', - 'UNIQUE(inscription_id, ref_inscription_genesis_id)' - ); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropConstraint('inscription_recursions', 'inscription_recursions_unique'); - pgm.dropIndex('inscription_recursions', ['ref_inscription_genesis_id']); - pgm.dropColumn('inscription_recursions', 'ref_inscription_genesis_id'); - pgm.dropIndex('inscription_recursions', ['ref_inscription_id'], { - name: 'inscription_recursions_ref_inscription_id_null_index', - }); - pgm.sql(`DELETE FROM inscription_recursions WHERE ref_inscription_id IS NULL`); - pgm.alterColumn('inscription_recursions', 'ref_inscription_id', { notNull: true }); - pgm.createConstraint( - 'inscription_recursions', - 'inscriptions_inscription_id_ref_inscription_id_unique', - 'UNIQUE(inscription_id, ref_inscription_id)' - ); -} diff --git a/migrations/1695655140203_counts-by-recursive.ts b/migrations/1695655140203_counts-by-recursive.ts index f19322af..b0fe6bc2 100644 --- a/migrations/1695655140203_counts-by-recursive.ts +++ b/migrations/1695655140203_counts-by-recursive.ts @@ -7,21 +7,12 @@ export function up(pgm: MigrationBuilder): void { pgm.createTable('counts_by_recursive', { recursive: { type: 'boolean', - notNull: true, primaryKey: true, }, count: { type: 'bigint', notNull: true, - default: 1, + default: 0, }, }); - pgm.sql(` - INSERT INTO counts_by_recursive (recursive, count) - (SELECT recursive, COUNT(*) AS count FROM inscriptions GROUP BY recursive) - `); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropTable('counts_by_recursive'); } diff --git a/migrations/1698897577725_locations-location-index.ts b/migrations/1698897577725_locations-location-index.ts deleted file mode 100644 index bb7461db..00000000 --- a/migrations/1698897577725_locations-location-index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.addColumn('locations', { - block_transfer_index: { - type: 'int', - }, - }); - pgm.addIndex('locations', ['block_height', { name: 'block_transfer_index', sort: 'DESC' }]); - pgm.addIndex('locations', ['block_hash', { name: 'block_transfer_index', sort: 'DESC' }]); -} diff --git a/migrations/1705363472553_locations-block-height-indexes.ts b/migrations/1705363472553_locations-block-height-indexes.ts deleted file mode 100644 index 304f3cac..00000000 --- a/migrations/1705363472553_locations-block-height-indexes.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropIndex('locations', ['block_hash']); - pgm.dropIndex('locations', ['block_height']); - pgm.createIndex('locations', [ - { name: 'block_height', sort: 'DESC' }, - { name: 'tx_index', sort: 'DESC' }, - ]); -} - -export function down(pgm: MigrationBuilder): void { - pgm.dropIndex('locations', [ - { name: 'block_height', sort: 'DESC' }, - { name: 'tx_index', sort: 'DESC' }, - ]); - pgm.createIndex('locations', ['block_hash']); - pgm.createIndex('locations', ['block_height']); -} diff --git a/migrations/1708471015438_remove-unused-indexes.ts b/migrations/1708471015438_remove-unused-indexes.ts deleted file mode 100644 index 1d94c6f7..00000000 --- a/migrations/1708471015438_remove-unused-indexes.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate'; - -export const shorthands: ColumnDefinitions | undefined = undefined; - -export function up(pgm: MigrationBuilder): void { - pgm.dropIndex('locations', ['prev_output']); - pgm.dropIndex('locations', ['address']); - pgm.dropIndex('current_locations', ['block_height']); - pgm.dropIndex('inscription_recursions', ['ref_inscription_genesis_id']); -} - -export function down(pgm: MigrationBuilder): void { - pgm.createIndex('locations', ['prev_output']); - pgm.createIndex('locations', ['address']); - pgm.createIndex('current_locations', ['block_height']); - pgm.createIndex('inscription_recursions', ['ref_inscription_genesis_id']); -} diff --git a/package-lock.json b/package-lock.json index 19c72ef7..d715c605 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "rimraf": "^3.0.2", "ts-jest": "^29.0.3", "ts-node": "^10.8.2", + "ts-unused-exports": "^10.0.1", "typescript": "^4.7.4" } }, @@ -18343,6 +18344,30 @@ } } }, + "node_modules/ts-unused-exports": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ts-unused-exports/-/ts-unused-exports-10.0.1.tgz", + "integrity": "sha512-nWG8Y96pKem01Hw4j4+Mwuy+L0/9sKT7D61Q+OS3cii9ocQACuV6lu00B9qpiPhF4ReVWw3QYHDqV8+to2wbsg==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0", + "tsconfig-paths": "^3.9.0" + }, + "bin": { + "ts-unused-exports": "bin/ts-unused-exports" + }, + "funding": { + "url": "https://github.com/pzavolinsky/ts-unused-exports?sponsor=1" + }, + "peerDependencies": { + "typescript": ">=3.8.3" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": false + } + } + }, "node_modules/tsconfig-paths": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", @@ -32335,6 +32360,16 @@ "yn": "3.1.1" } }, + "ts-unused-exports": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ts-unused-exports/-/ts-unused-exports-10.0.1.tgz", + "integrity": "sha512-nWG8Y96pKem01Hw4j4+Mwuy+L0/9sKT7D61Q+OS3cii9ocQACuV6lu00B9qpiPhF4ReVWw3QYHDqV8+to2wbsg==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "tsconfig-paths": "^3.9.0" + } + }, "tsconfig-paths": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", diff --git a/package.json b/package.json index 132e5ff8..889db34b 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "migrate": "ts-node node_modules/.bin/node-pg-migrate -j ts", "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f unix", "lint:prettier": "prettier --check src/**/*.ts tests/**/*.ts migrations/**/*.ts", + "lint:unused-exports": "ts-unused-exports tsconfig.json --showLineNumber --excludePathsFromReport=migrations/* --excludePathsFromReport=util/*", "generate:openapi": "rimraf ./tmp && node -r ts-node/register ./util/openapi-generator.ts", "generate:docs": "redoc-cli build --output ./tmp/index.html ./tmp/openapi.yaml", "generate:git-info": "rimraf .git-info && node_modules/.bin/api-toolkit-git-info", @@ -46,6 +47,7 @@ "rimraf": "^3.0.2", "ts-jest": "^29.0.3", "ts-node": "^10.8.2", + "ts-unused-exports": "^10.0.1", "typescript": "^4.7.4" }, "dependencies": { diff --git a/src/api/routes/stats.ts b/src/api/routes/stats.ts index f5539bb6..94c64a3d 100644 --- a/src/api/routes/stats.ts +++ b/src/api/routes/stats.ts @@ -31,7 +31,7 @@ const IndexRoutes: FastifyPluginCallback, Server, TypeBoxTy }, }, async (request, reply) => { - const inscriptions = await fastify.db.getInscriptionCountPerBlock({ + const inscriptions = await fastify.db.counts.getInscriptionCountPerBlock({ ...blockParam(request.query.from_block_height, 'from_block'), ...blockParam(request.query.to_block_height, 'to_block'), }); diff --git a/src/api/schemas.ts b/src/api/schemas.ts index 053f38d3..21483605 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -82,7 +82,7 @@ export const Brc20TickerParam = Type.String(); export const Brc20TickersParam = Type.Array(Brc20TickerParam); -export const InscriptionIdParam = Type.RegEx(/^[a-fA-F0-9]{64}i[0-9]+$/, { +const InscriptionIdParam = Type.RegEx(/^[a-fA-F0-9]{64}i[0-9]+$/, { title: 'Inscription ID', description: 'Inscription ID', examples: ['38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0'], @@ -134,7 +134,7 @@ export const BlockHeightParam = Type.RegEx(/^[0-9]+$/, { }); export const BlockHeightParamCType = TypeCompiler.Compile(BlockHeightParam); -export const BlockHashParam = Type.RegEx(/^[0]{8}[a-fA-F0-9]{56}$/, { +const BlockHashParam = Type.RegEx(/^[0]{8}[a-fA-F0-9]{56}$/, { title: 'Block Hash', description: 'Bitcoin block hash', examples: ['0000000000000000000452773967cdd62297137cdaf79950c5e8bb0c62075133'], @@ -210,7 +210,7 @@ export const LimitParam = Type.Integer({ description: 'Results per page', }); -export const Brc20OperationParam = Type.Union( +const Brc20OperationParam = Type.Union( [ Type.Literal('deploy'), Type.Literal('mint'), @@ -494,7 +494,7 @@ export const Brc20TokenResponseSchema = Type.Object( ); export type Brc20TokenResponse = Static; -export const Brc20SupplySchema = Type.Object({ +const Brc20SupplySchema = Type.Object({ max_supply: Type.String({ examples: ['21000000'] }), minted_supply: Type.String({ examples: ['1000000'] }), holders: Type.Integer({ examples: [240] }), @@ -516,7 +516,7 @@ export const Brc20TokenDetailsSchema = Type.Object( }, { title: 'BRC-20 Token Details Response' } ); -export type Brc20TokenDetails = Static; +type Brc20TokenDetails = Static; export const NotFoundResponse = Type.Object( { @@ -532,7 +532,7 @@ export const InvalidSatoshiNumberResponse = Type.Object( { title: 'Invalid Satoshi Number Response' } ); -export const InscriptionsPerBlock = Type.Object({ +const InscriptionsPerBlock = Type.Object({ block_height: Type.String({ examples: ['778921'] }), block_hash: Type.String({ examples: ['0000000000000000000452773967cdd62297137cdaf79950c5e8bb0c62075133'], diff --git a/src/api/util/cache.ts b/src/api/util/cache.ts index cc08159e..c69e0e9d 100644 --- a/src/api/util/cache.ts +++ b/src/api/util/cache.ts @@ -2,7 +2,7 @@ import { FastifyReply, FastifyRequest } from 'fastify'; import { InscriptionIdParamCType, InscriptionNumberParamCType } from '../schemas'; import { logger } from '@hirosystems/api-toolkit'; -export enum ETagType { +enum ETagType { inscriptionsIndex, inscription, inscriptionsPerBlock, @@ -57,11 +57,6 @@ async function handleCache(type: ETagType, request: FastifyRequest, reply: Fasti } } -export function setReplyNonCacheable(reply: FastifyReply) { - reply.removeHeader('Cache-Control'); - reply.removeHeader('Etag'); -} - /** * Retrieve the inscriptions's location timestamp as a UNIX epoch so we can use it as the response * ETag. diff --git a/src/api/util/helpers.ts b/src/api/util/helpers.ts index 9ff8c520..0b51af1c 100644 --- a/src/api/util/helpers.ts +++ b/src/api/util/helpers.ts @@ -233,7 +233,7 @@ export function hexToBuffer(hex: string): Buffer { return Buffer.from(hex.substring(2), 'hex'); } -export const has0xPrefix = (id: string) => id.substr(0, 2).toLowerCase() === '0x'; +const has0xPrefix = (id: string) => id.substr(0, 2).toLowerCase() === '0x'; export function normalizedHexString(hex: string): string { return has0xPrefix(hex) ? hex.substring(2) : hex; diff --git a/src/ordhook/server.ts b/src/ordhook/server.ts index 7f604584..19d1decc 100644 --- a/src/ordhook/server.ts +++ b/src/ordhook/server.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'crypto'; import { ENV } from '../env'; import { PgStore } from '../pg/pg-store'; import { + BitcoinPayload, ChainhookEventObserver, ChainhookNodeOptions, Payload, @@ -64,7 +65,7 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise(); + recursiveRefs = new Map(); + + mimeTypeCounts = new Map(); + satRarityCounts = new Map(); + inscriptionTypeCounts = new Map(); + genesisAddressCounts = new Map(); + recursiveCounts = new Map(); + + constructor(blockHeight: number, blockHash: string, timestamp: number) { + this.blockHeight = blockHeight; + this.blockHash = blockHash; + this.timestamp = timestamp; + } + + reveal(reveal: BitcoinInscriptionRevealed, tx_id: string) { + const satoshi = new OrdinalSatoshi(reveal.ordinal_number); + const ordinal_number = reveal.ordinal_number.toString(); + this.satoshis.push({ + ordinal_number, + rarity: satoshi.rarity, + coinbase_height: satoshi.blockHeight, + }); + const satpoint = parseSatPoint(reveal.satpoint_post_inscription); + const recursive_refs = getInscriptionRecursion(reveal.content_bytes); + const content_type = removeNullBytes(reveal.content_type); + const mime_type = content_type.split(';')[0]; + this.inscriptions.push({ + genesis_id: reveal.inscription_id, + mime_type, + content_type, + content_length: reveal.content_length, + block_height: this.blockHeight, + tx_index: reveal.tx_index, + address: reveal.inscriber_address, + number: reveal.inscription_number.jubilee, + classic_number: reveal.inscription_number.classic, + content: removeNullBytes(reveal.content_bytes), + fee: reveal.inscription_fee.toString(), + curse_type: reveal.curse_type ? JSON.stringify(reveal.curse_type) : null, + ordinal_number, + recursive: recursive_refs.length > 0, + metadata: reveal.metadata ? JSON.stringify(reveal.metadata) : null, + parent: reveal.parent, + timestamp: this.timestamp, + }); + this.increaseMimeTypeCount(mime_type); + this.increaseSatRarityCount(satoshi.rarity); + this.increaseInscriptionTypeCount(reveal.inscription_number.classic < 0 ? 'cursed' : 'blessed'); + this.increaseGenesisAddressCount(reveal.inscriber_address); + this.increaseRecursiveCount(recursive_refs.length > 0); + this.locations.push({ + block_hash: this.blockHash, + block_height: this.blockHeight, + tx_id, + tx_index: reveal.tx_index, + ordinal_number, + address: reveal.inscriber_address, + output: `${satpoint.tx_id}:${satpoint.vout}`, + offset: satpoint.offset ?? null, + prev_output: null, + prev_offset: null, + value: reveal.inscription_output_value.toString(), + timestamp: this.timestamp, + transfer_type: getTransferType(reveal), + }); + this.updateCurrentLocation(ordinal_number, { + ordinal_number, + block_height: this.blockHeight, + tx_index: reveal.tx_index, + address: reveal.inscriber_address, + }); + if (recursive_refs.length > 0) this.recursiveRefs.set(reveal.inscription_id, recursive_refs); + } + + transfer(transfer: BitcoinInscriptionTransferred, tx_id: string) { + const satpoint = parseSatPoint(transfer.satpoint_post_transfer); + const prevSatpoint = parseSatPoint(transfer.satpoint_pre_transfer); + const ordinal_number = transfer.ordinal_number.toString(); + const address = transfer.destination.value ?? null; + this.locations.push({ + block_hash: this.blockHash, + block_height: this.blockHeight, + tx_id, + tx_index: transfer.tx_index, + ordinal_number, + address, + output: `${satpoint.tx_id}:${satpoint.vout}`, + offset: satpoint.offset ?? null, + prev_output: `${prevSatpoint.tx_id}:${prevSatpoint.vout}`, + prev_offset: prevSatpoint.offset ?? null, + value: transfer.post_transfer_output_value + ? transfer.post_transfer_output_value.toString() + : null, + timestamp: this.timestamp, + transfer_type: + toEnumValue(DbLocationTransferType, transfer.destination.type) ?? + DbLocationTransferType.transferred, + }); + this.updateCurrentLocation(ordinal_number, { + ordinal_number, + block_height: this.blockHeight, + tx_index: transfer.tx_index, + address, + }); + } + + private updateCurrentLocation(ordinal_number: string, data: DbCurrentLocationInsert) { + const current = this.currentLocations.get(ordinal_number); + if ( + current === undefined || + (current && + (data.block_height > current.block_height || + (data.block_height === current.block_height && data.tx_index > current.tx_index))) + ) { + this.currentLocations.set(ordinal_number, data); + } + } + + private increaseMimeTypeCount(mime_type: string) { + const current = this.mimeTypeCounts.get(mime_type); + if (current == undefined) { + this.mimeTypeCounts.set(mime_type, 1); + } else { + this.mimeTypeCounts.set(mime_type, current + 1); + } + } + + private increaseSatRarityCount(rarity: string) { + const current = this.satRarityCounts.get(rarity); + if (current == undefined) { + this.satRarityCounts.set(rarity, 1); + } else { + this.satRarityCounts.set(rarity, current + 1); + } + } + + private increaseInscriptionTypeCount(type: string) { + const current = this.inscriptionTypeCounts.get(type); + if (current == undefined) { + this.inscriptionTypeCounts.set(type, 1); + } else { + this.inscriptionTypeCounts.set(type, current + 1); + } + } + + private increaseGenesisAddressCount(address: string | null) { + if (!address) return; + const current = this.genesisAddressCounts.get(address); + if (current == undefined) { + this.genesisAddressCounts.set(address, 1); + } else { + this.genesisAddressCounts.set(address, current + 1); + } + } + + private increaseRecursiveCount(recursive: boolean) { + const current = this.recursiveCounts.get(recursive); + if (current == undefined) { + this.recursiveCounts.set(recursive, 1); + } else { + this.recursiveCounts.set(recursive, current + 1); + } + } +} diff --git a/src/pg/brc20/brc20-block-cache.ts b/src/pg/brc20/brc20-block-cache.ts new file mode 100644 index 00000000..3bd35659 --- /dev/null +++ b/src/pg/brc20/brc20-block-cache.ts @@ -0,0 +1,228 @@ +import BigNumber from 'bignumber.js'; +import { DbBrc20TokenInsert, DbBrc20OperationInsert, DbBrc20Operation } from './types'; +import { + BitcoinBrc20DeployOperation, + BitcoinBrc20MintOperation, + BitcoinBrc20TransferOperation, + BitcoinBrc20TransferSendOperation, +} from '@hirosystems/chainhook-client'; + +interface AddressBalanceData { + avail: BigNumber; + trans: BigNumber; + total: BigNumber; +} + +/** + * In-memory cache for an Ordhook block's BRC-20 activities. + */ +export class Brc20BlockCache { + blockHeight: number; + + tokens: DbBrc20TokenInsert[] = []; + operations: DbBrc20OperationInsert[] = []; + tokenMintSupplies = new Map(); + tokenTxCounts = new Map(); + operationCounts = new Map(); + addressOperationCounts = new Map>(); + totalBalanceChanges = new Map>(); + transferReceivers = new Map(); + + constructor(blockHeight: number) { + this.blockHeight = blockHeight; + } + + deploy(operation: BitcoinBrc20DeployOperation, tx_id: string, tx_index: number) { + this.tokens.push({ + block_height: this.blockHeight, + genesis_id: operation.deploy.inscription_id, + tx_id, + address: operation.deploy.address, + ticker: operation.deploy.tick, + max: operation.deploy.max, + limit: operation.deploy.lim, + decimals: operation.deploy.dec, + self_mint: operation.deploy.self_mint, + }); + this.operations.push({ + block_height: this.blockHeight, + tx_index, + genesis_id: operation.deploy.inscription_id, + ticker: operation.deploy.tick, + address: operation.deploy.address, + avail_balance: '0', + trans_balance: '0', + operation: DbBrc20Operation.deploy, + }); + this.increaseOperationCount(DbBrc20Operation.deploy); + this.increaseAddressOperationCount(operation.deploy.address, DbBrc20Operation.deploy); + this.increaseTokenTxCount(operation.deploy.tick); + } + + mint(operation: BitcoinBrc20MintOperation, tx_index: number) { + this.operations.push({ + block_height: this.blockHeight, + tx_index, + genesis_id: operation.mint.inscription_id, + ticker: operation.mint.tick, + address: operation.mint.address, + avail_balance: operation.mint.amt, + trans_balance: '0', + operation: DbBrc20Operation.mint, + }); + const amt = BigNumber(operation.mint.amt); + this.increaseTokenMintedSupply(operation.mint.tick, amt); + this.increaseTokenTxCount(operation.mint.tick); + this.increaseOperationCount(DbBrc20Operation.mint); + this.increaseAddressOperationCount(operation.mint.address, DbBrc20Operation.mint); + this.updateAddressBalance(operation.mint.tick, operation.mint.address, amt, BigNumber(0), amt); + } + + transfer(operation: BitcoinBrc20TransferOperation, tx_index: number) { + this.operations.push({ + block_height: this.blockHeight, + tx_index, + genesis_id: operation.transfer.inscription_id, + ticker: operation.transfer.tick, + address: operation.transfer.address, + avail_balance: BigNumber(operation.transfer.amt).negated().toString(), + trans_balance: operation.transfer.amt, + operation: DbBrc20Operation.transfer, + }); + const amt = BigNumber(operation.transfer.amt); + this.increaseOperationCount(DbBrc20Operation.transfer); + this.increaseTokenTxCount(operation.transfer.tick); + this.increaseAddressOperationCount(operation.transfer.address, DbBrc20Operation.transfer); + this.updateAddressBalance( + operation.transfer.tick, + operation.transfer.address, + amt.negated(), + amt, + BigNumber(0) + ); + } + + transferSend(operation: BitcoinBrc20TransferSendOperation, tx_index: number) { + this.operations.push({ + block_height: this.blockHeight, + tx_index, + genesis_id: operation.transfer_send.inscription_id, + ticker: operation.transfer_send.tick, + address: operation.transfer_send.sender_address, + avail_balance: '0', + trans_balance: BigNumber(operation.transfer_send.amt).negated().toString(), + operation: DbBrc20Operation.transferSend, + }); + this.transferReceivers.set( + operation.transfer_send.inscription_id, + operation.transfer_send.receiver_address + ); + this.operations.push({ + block_height: this.blockHeight, + tx_index, + genesis_id: operation.transfer_send.inscription_id, + ticker: operation.transfer_send.tick, + address: operation.transfer_send.receiver_address, + avail_balance: operation.transfer_send.amt, + trans_balance: '0', + operation: DbBrc20Operation.transferReceive, + }); + const amt = BigNumber(operation.transfer_send.amt); + this.increaseOperationCount(DbBrc20Operation.transferSend); + this.increaseTokenTxCount(operation.transfer_send.tick); + this.increaseAddressOperationCount( + operation.transfer_send.sender_address, + DbBrc20Operation.transferSend + ); + if (operation.transfer_send.sender_address != operation.transfer_send.receiver_address) { + this.increaseAddressOperationCount( + operation.transfer_send.receiver_address, + DbBrc20Operation.transferSend + ); + } + this.updateAddressBalance( + operation.transfer_send.tick, + operation.transfer_send.sender_address, + BigNumber('0'), + amt.negated(), + amt.negated() + ); + this.updateAddressBalance( + operation.transfer_send.tick, + operation.transfer_send.receiver_address, + amt, + BigNumber(0), + amt + ); + } + + private increaseOperationCount(operation: DbBrc20Operation) { + this.increaseOperationCountInternal(this.operationCounts, operation); + } + private increaseOperationCountInternal( + map: Map, + operation: DbBrc20Operation + ) { + const current = map.get(operation); + if (current == undefined) { + map.set(operation, 1); + } else { + map.set(operation, current + 1); + } + } + + private increaseTokenMintedSupply(ticker: string, amount: BigNumber) { + const current = this.tokenMintSupplies.get(ticker); + if (current == undefined) { + this.tokenMintSupplies.set(ticker, amount); + } else { + this.tokenMintSupplies.set(ticker, current.plus(amount)); + } + } + + private increaseTokenTxCount(ticker: string) { + const current = this.tokenTxCounts.get(ticker); + if (current == undefined) { + this.tokenTxCounts.set(ticker, 1); + } else { + this.tokenTxCounts.set(ticker, current + 1); + } + } + + private increaseAddressOperationCount(address: string, operation: DbBrc20Operation) { + const current = this.addressOperationCounts.get(address); + if (current == undefined) { + const opMap = new Map(); + this.increaseOperationCountInternal(opMap, operation); + this.addressOperationCounts.set(address, opMap); + } else { + this.increaseOperationCountInternal(current, operation); + } + } + + private updateAddressBalance( + ticker: string, + address: string, + availBalance: BigNumber, + transBalance: BigNumber, + totalBalance: BigNumber + ) { + const current = this.totalBalanceChanges.get(address); + if (current === undefined) { + const opMap = new Map(); + opMap.set(ticker, { avail: availBalance, trans: transBalance, total: totalBalance }); + this.totalBalanceChanges.set(address, opMap); + } else { + const currentTick = current.get(ticker); + if (currentTick === undefined) { + current.set(ticker, { avail: availBalance, trans: transBalance, total: totalBalance }); + } else { + current.set(ticker, { + avail: availBalance.plus(currentTick.avail), + trans: transBalance.plus(currentTick.trans), + total: totalBalance.plus(currentTick.total), + }); + } + } + } +} diff --git a/src/pg/brc20/brc20-pg-store.ts b/src/pg/brc20/brc20-pg-store.ts index 1699e041..5f727507 100644 --- a/src/pg/brc20/brc20-pg-store.ts +++ b/src/pg/brc20/brc20-pg-store.ts @@ -6,169 +6,58 @@ import { DbBrc20Holder, DbBrc20Token, DbBrc20TokenWithSupply, - DbBrc20Operation, } from './types'; import { Brc20TokenOrderBy } from '../../api/schemas'; import { objRemoveUndefinedValues } from '../helpers'; import { BitcoinEvent } from '@hirosystems/chainhook-client'; -import BigNumber from 'bignumber.js'; -import { Brc20BlockCache, sqlOr } from './helpers'; +import { sqlOr } from './helpers'; import { INSERT_BATCH_SIZE } from '../pg-store'; +import { Brc20BlockCache } from './brc20-block-cache'; export class Brc20PgStore extends BasePgStoreModule { - async updateBrc20Operations(event: BitcoinEvent, direction: 'apply' | 'rollback'): Promise { - await this.sqlWriteTransaction(async sql => { - const block_height = event.block_identifier.index.toString(); - const cache = new Brc20BlockCache(); - for (const tx of event.transactions) { - const tx_index = tx.metadata.index.toString(); - if (tx.metadata.brc20_operation) { - const operation = tx.metadata.brc20_operation; - if ('deploy' in operation) { - cache.tokens.push({ - block_height, - genesis_id: operation.deploy.inscription_id, - tx_id: tx.transaction_identifier.hash, - address: operation.deploy.address, - ticker: operation.deploy.tick, - max: operation.deploy.max, - limit: operation.deploy.lim, - decimals: operation.deploy.dec, - self_mint: operation.deploy.self_mint, - }); - cache.operations.push({ - block_height, - tx_index, - genesis_id: operation.deploy.inscription_id, - ticker: operation.deploy.tick, - address: operation.deploy.address, - avail_balance: '0', - trans_balance: '0', - operation: DbBrc20Operation.deploy, - }); - cache.increaseOperationCount(DbBrc20Operation.deploy); - cache.increaseAddressOperationCount(operation.deploy.address, DbBrc20Operation.deploy); - cache.increaseTokenTxCount(operation.deploy.tick); - logger.info( - `Brc20PgStore ${direction} deploy ${operation.deploy.tick} by ${operation.deploy.address} at height ${block_height}` - ); - } else if ('mint' in operation) { - cache.operations.push({ - block_height, - tx_index, - genesis_id: operation.mint.inscription_id, - ticker: operation.mint.tick, - address: operation.mint.address, - avail_balance: operation.mint.amt, - trans_balance: '0', - operation: DbBrc20Operation.mint, - }); - const amt = BigNumber(operation.mint.amt); - cache.increaseTokenMintedSupply(operation.mint.tick, amt); - cache.increaseTokenTxCount(operation.mint.tick); - cache.increaseOperationCount(DbBrc20Operation.mint); - cache.increaseAddressOperationCount(operation.mint.address, DbBrc20Operation.mint); - cache.updateAddressBalance( - operation.mint.tick, - operation.mint.address, - amt, - BigNumber(0), - amt - ); - logger.info( - `Brc20PgStore ${direction} mint ${operation.mint.tick} ${operation.mint.amt} by ${operation.mint.address} at height ${block_height}` - ); - } else if ('transfer' in operation) { - cache.operations.push({ - block_height, - tx_index, - genesis_id: operation.transfer.inscription_id, - ticker: operation.transfer.tick, - address: operation.transfer.address, - avail_balance: BigNumber(operation.transfer.amt).negated().toString(), - trans_balance: operation.transfer.amt, - operation: DbBrc20Operation.transfer, - }); - const amt = BigNumber(operation.transfer.amt); - cache.increaseOperationCount(DbBrc20Operation.transfer); - cache.increaseTokenTxCount(operation.transfer.tick); - cache.increaseAddressOperationCount( - operation.transfer.address, - DbBrc20Operation.transfer - ); - cache.updateAddressBalance( - operation.transfer.tick, - operation.transfer.address, - amt.negated(), - amt, - BigNumber(0) - ); - logger.info( - `Brc20PgStore ${direction} transfer ${operation.transfer.tick} ${operation.transfer.amt} by ${operation.transfer.address} at height ${block_height}` - ); - } else if ('transfer_send' in operation) { - cache.operations.push({ - block_height, - tx_index, - genesis_id: operation.transfer_send.inscription_id, - ticker: operation.transfer_send.tick, - address: operation.transfer_send.sender_address, - avail_balance: '0', - trans_balance: BigNumber(operation.transfer_send.amt).negated().toString(), - operation: DbBrc20Operation.transferSend, - }); - cache.transferReceivers.set( - operation.transfer_send.inscription_id, - operation.transfer_send.receiver_address - ); - cache.operations.push({ - block_height, - tx_index, - genesis_id: operation.transfer_send.inscription_id, - ticker: operation.transfer_send.tick, - address: operation.transfer_send.receiver_address, - avail_balance: operation.transfer_send.amt, - trans_balance: '0', - operation: DbBrc20Operation.transferReceive, - }); - const amt = BigNumber(operation.transfer_send.amt); - cache.increaseOperationCount(DbBrc20Operation.transferSend); - cache.increaseTokenTxCount(operation.transfer_send.tick); - cache.increaseAddressOperationCount( - operation.transfer_send.sender_address, - DbBrc20Operation.transferSend - ); - if ( - operation.transfer_send.sender_address != operation.transfer_send.receiver_address - ) { - cache.increaseAddressOperationCount( - operation.transfer_send.receiver_address, - DbBrc20Operation.transferSend - ); - } - cache.updateAddressBalance( - operation.transfer_send.tick, - operation.transfer_send.sender_address, - BigNumber('0'), - amt.negated(), - amt.negated() - ); - cache.updateAddressBalance( - operation.transfer_send.tick, - operation.transfer_send.receiver_address, - amt, - BigNumber(0), - amt - ); - logger.info( - `Brc20PgStore ${direction} transfer_send ${operation.transfer_send.tick} ${operation.transfer_send.amt} from ${operation.transfer_send.sender_address} to ${operation.transfer_send.receiver_address} at height ${block_height}` - ); - } + async updateBrc20Operations( + sql: PgSqlClient, + event: BitcoinEvent, + direction: 'apply' | 'rollback' + ): Promise { + const block_height = event.block_identifier.index; + const cache = new Brc20BlockCache(block_height); + for (const tx of event.transactions) { + const tx_id = tx.transaction_identifier.hash; + const tx_index = tx.metadata.index; + if (tx.metadata.brc20_operation) { + const operation = tx.metadata.brc20_operation; + if ('deploy' in operation) { + cache.deploy(operation, tx_id, tx_index); + logger.info( + `Brc20PgStore ${direction} deploy ${operation.deploy.tick} by ${operation.deploy.address} at height ${block_height}` + ); + } else if ('mint' in operation) { + cache.mint(operation, tx_index); + logger.info( + `Brc20PgStore ${direction} mint ${operation.mint.tick} ${operation.mint.amt} by ${operation.mint.address} at height ${block_height}` + ); + } else if ('transfer' in operation) { + cache.transfer(operation, tx_index); + logger.info( + `Brc20PgStore ${direction} transfer ${operation.transfer.tick} ${operation.transfer.amt} by ${operation.transfer.address} at height ${block_height}` + ); + } else if ('transfer_send' in operation) { + cache.transferSend(operation, tx_index); + logger.info( + `Brc20PgStore ${direction} transfer_send ${operation.transfer_send.tick} ${operation.transfer_send.amt} from ${operation.transfer_send.sender_address} to ${operation.transfer_send.receiver_address} at height ${block_height}` + ); } } - if (direction === 'apply') await this.applyOperations(sql, cache); - else await this.rollBackOperations(sql, cache); - }); + } + switch (direction) { + case 'apply': + await this.applyOperations(sql, cache); + break; + case 'rollback': + await this.rollBackOperations(sql, cache); + break; + } } private async applyOperations(sql: PgSqlClient, cache: Brc20BlockCache) { @@ -310,7 +199,7 @@ export class Brc20PgStore extends BasePgStoreModule { const orderBy = args.order_by === Brc20TokenOrderBy.tx_count ? this.sql`d.tx_count DESC` // tx_count - : this.sql`l.block_height DESC, l.tx_index DESC`; // default: `index` + : this.sql`i.block_height DESC, i.tx_index DESC`; // default: `index` const results = await this.sql<(DbBrc20Token & { total: number })[]>` ${ args.ticker === undefined @@ -322,14 +211,12 @@ export class Brc20PgStore extends BasePgStoreModule { : this.sql`` } SELECT - d.*, i.number, l.timestamp, + d.*, i.number, i.timestamp, ${ args.ticker ? this.sql`COUNT(*) OVER()` : this.sql`(SELECT count FROM global_count)` } AS total FROM brc20_tokens AS d INNER JOIN inscriptions AS i ON i.genesis_id = d.genesis_id - INNER JOIN genesis_locations AS g ON g.inscription_id = i.id - INNER JOIN locations AS l ON l.id = g.location_id ${tickerPrefixCondition ? this.sql`WHERE ${tickerPrefixCondition}` : this.sql``} ORDER BY ${orderBy} OFFSET ${args.offset} @@ -395,11 +282,9 @@ export class Brc20PgStore extends BasePgStoreModule { const result = await this.sql` WITH token AS ( SELECT - d.*, i.number, i.genesis_id, l.timestamp + d.*, i.number, i.genesis_id, i.timestamp FROM brc20_tokens AS d INNER JOIN inscriptions AS i ON i.genesis_id = d.genesis_id - INNER JOIN genesis_locations AS g ON g.inscription_id = i.id - INNER JOIN locations AS l ON l.id = g.location_id WHERE d.ticker = LOWER(${args.ticker}) ), holders AS ( @@ -496,8 +381,8 @@ export class Brc20PgStore extends BasePgStoreModule { e.address, e.to_address, d.ticker, - l.genesis_id AS inscription_id, - l.block_height, + e.genesis_id AS inscription_id, + i.block_height, l.block_hash, l.tx_id, l.timestamp, @@ -513,7 +398,8 @@ export class Brc20PgStore extends BasePgStoreModule { } AS total FROM brc20_operations AS e INNER JOIN brc20_tokens AS d ON d.ticker = e.ticker - INNER JOIN locations AS l ON e.genesis_id = l.genesis_id AND e.block_height = l.block_height AND e.tx_index = l.tx_index + INNER JOIN inscriptions AS i ON i.genesis_id = e.genesis_id + INNER JOIN locations AS l ON i.ordinal_number = l.ordinal_number AND e.block_height = l.block_height AND e.tx_index = l.tx_index WHERE TRUE ${ operationsFilter @@ -521,7 +407,7 @@ export class Brc20PgStore extends BasePgStoreModule { : sql`AND e.operation <> 'transfer_receive'` } ${filters.ticker ? sql`AND e.ticker IN ${sql(filters.ticker)}` : sql``} - ${filters.block_height ? sql`AND l.block_height = ${filters.block_height}` : sql``} + ${filters.block_height ? sql`AND e.block_height = ${filters.block_height}` : sql``} ${ filters.address ? sql`AND (e.address = ${filters.address} OR e.to_address = ${filters.address})` diff --git a/src/pg/brc20/helpers.ts b/src/pg/brc20/helpers.ts index 1d04e767..b0e9e6c1 100644 --- a/src/pg/brc20/helpers.ts +++ b/src/pg/brc20/helpers.ts @@ -1,5 +1,3 @@ -import BigNumber from 'bignumber.js'; -import { DbBrc20Operation, DbBrc20OperationInsert, DbBrc20TokenInsert } from './types'; import * as postgres from 'postgres'; import { PgSqlClient } from '@hirosystems/api-toolkit'; @@ -9,90 +7,3 @@ export function sqlOr( ) { return partials?.reduce((acc, curr) => sql`${acc} OR ${curr}`); } - -export interface AddressBalanceData { - avail: BigNumber; - trans: BigNumber; - total: BigNumber; -} - -export class Brc20BlockCache { - tokens: DbBrc20TokenInsert[] = []; - operations: DbBrc20OperationInsert[] = []; - tokenMintSupplies = new Map(); - tokenTxCounts = new Map(); - operationCounts = new Map(); - addressOperationCounts = new Map>(); - totalBalanceChanges = new Map>(); - transferReceivers = new Map(); - - increaseOperationCount(operation: DbBrc20Operation) { - this.increaseOperationCountInternal(this.operationCounts, operation); - } - private increaseOperationCountInternal( - map: Map, - operation: DbBrc20Operation - ) { - const current = map.get(operation); - if (current == undefined) { - map.set(operation, 1); - } else { - map.set(operation, current + 1); - } - } - - increaseTokenMintedSupply(ticker: string, amount: BigNumber) { - const current = this.tokenMintSupplies.get(ticker); - if (current == undefined) { - this.tokenMintSupplies.set(ticker, amount); - } else { - this.tokenMintSupplies.set(ticker, current.plus(amount)); - } - } - - increaseTokenTxCount(ticker: string) { - const current = this.tokenTxCounts.get(ticker); - if (current == undefined) { - this.tokenTxCounts.set(ticker, 1); - } else { - this.tokenTxCounts.set(ticker, current + 1); - } - } - - increaseAddressOperationCount(address: string, operation: DbBrc20Operation) { - const current = this.addressOperationCounts.get(address); - if (current == undefined) { - const opMap = new Map(); - this.increaseOperationCountInternal(opMap, operation); - this.addressOperationCounts.set(address, opMap); - } else { - this.increaseOperationCountInternal(current, operation); - } - } - - updateAddressBalance( - ticker: string, - address: string, - availBalance: BigNumber, - transBalance: BigNumber, - totalBalance: BigNumber - ) { - const current = this.totalBalanceChanges.get(address); - if (current === undefined) { - const opMap = new Map(); - opMap.set(ticker, { avail: availBalance, trans: transBalance, total: totalBalance }); - this.totalBalanceChanges.set(address, opMap); - } else { - const currentTick = current.get(ticker); - if (currentTick === undefined) { - current.set(ticker, { avail: availBalance, trans: transBalance, total: totalBalance }); - } else { - current.set(ticker, { - avail: availBalance.plus(currentTick.avail), - trans: transBalance.plus(currentTick.trans), - total: totalBalance.plus(currentTick.total), - }); - } - } - } -} diff --git a/src/pg/brc20/types.ts b/src/pg/brc20/types.ts index 9737dfb6..c3bb8a32 100644 --- a/src/pg/brc20/types.ts +++ b/src/pg/brc20/types.ts @@ -3,7 +3,7 @@ import { PgNumeric } from '@hirosystems/api-toolkit'; export type DbBrc20TokenInsert = { ticker: string; genesis_id: string; - block_height: string; + block_height: number; tx_id: string; address: string; max: PgNumeric; @@ -23,20 +23,14 @@ export enum DbBrc20Operation { export type DbBrc20OperationInsert = { genesis_id: string; ticker: string; - block_height: PgNumeric; - tx_index: PgNumeric; + block_height: number; + tx_index: number; address: string; avail_balance: PgNumeric; trans_balance: PgNumeric; operation: DbBrc20Operation; }; -export type DbBrc20CountsByAddressInsert = { - address: string; - operation: DbBrc20Operation; - count: number; -}; - export type DbBrc20Token = { id: string; genesis_id: string; @@ -73,49 +67,12 @@ export type DbBrc20Balance = { total_balance: string; }; -export enum DbBrc20BalanceTypeId { - mint = 0, - transferIntent = 1, - transferFrom = 2, - transferTo = 3, -} - export enum DbBrc20EventOperation { deploy = 'deploy', mint = 'mint', transfer = 'transfer', transferSend = 'transfer_send', } -export const BRC20_OPERATIONS = ['deploy', 'mint', 'transfer', 'transfer_send']; - -type BaseEvent = { - inscription_id: string; - genesis_location_id: string; - brc20_deploy_id: string; -}; - -export type DbBrc20DeployEvent = BaseEvent & { - operation: 'deploy'; - deploy_id: string; - mint_id: null; - transfer_id: null; -}; - -export type DbBrc20MintEvent = BaseEvent & { - operation: 'mint'; - deploy_id: null; - mint_id: string; - transfer_id: null; -}; - -export type DbBrc20TransferEvent = BaseEvent & { - operation: 'transfer' | 'transfer_send'; - deploy_id: null; - mint_id: null; - transfer_id: string; -}; - -export type DbBrc20Event = DbBrc20DeployEvent | DbBrc20MintEvent | DbBrc20TransferEvent; export type DbBrc20Activity = { ticker: string; @@ -136,18 +93,3 @@ export type DbBrc20Activity = { to_address: string | null; timestamp: number; }; - -export const BRC20_DEPLOYS_COLUMNS = [ - 'id', - 'inscription_id', - 'block_height', - 'tx_id', - 'address', - 'ticker', - 'max', - 'decimals', - 'limit', - 'minted_supply', - 'tx_count', - 'self_mint', -]; diff --git a/src/pg/counts/counts-pg-store.ts b/src/pg/counts/counts-pg-store.ts index 6994959c..7e56e54c 100644 --- a/src/pg/counts/counts-pg-store.ts +++ b/src/pg/counts/counts-pg-store.ts @@ -1,14 +1,13 @@ -import { BasePgStoreModule } from '@hirosystems/api-toolkit'; +import { BasePgStoreModule, PgSqlClient } from '@hirosystems/api-toolkit'; import { SatoshiRarity } from '../../api/util/ordinal-satoshi'; import { - DbInscription, + DbInscriptionCountPerBlock, + DbInscriptionCountPerBlockFilters, DbInscriptionIndexFilters, - InscriptionData, DbInscriptionType, - RevealLocationData, - DbLocationPointer, } from '../types'; import { DbInscriptionIndexResultCountType } from './types'; +import { BlockCache } from '../block-cache'; /** * This class affects all the different tables that track inscription counts according to different @@ -55,142 +54,128 @@ export class CountsPgStore extends BasePgStoreModule { } } - async applyInscriptions(writes: InscriptionData[]): Promise { - if (writes.length === 0) return; - const mimeType = new Map(); - const rarity = new Map(); - const recursion = new Map(); - const typeMap = new Map(); - for (const i of writes) { - mimeType.set(i.mime_type, (mimeType.get(i.mime_type) ?? 0) + 1); - rarity.set(i.sat_rarity, (rarity.get(i.sat_rarity) ?? 0) + 1); - recursion.set(i.recursive, (recursion.get(i.recursive) ?? 0) + 1); - const inscrType = i.number < 0 ? 'cursed' : 'blessed'; - typeMap.set(inscrType, (typeMap.get(inscrType) ?? 0) + 1); - } - const mimeTypeInsert = Array.from(mimeType.entries()).map(k => ({ - mime_type: k[0], - count: k[1], - })); - const rarityInsert = Array.from(rarity.entries()).map(k => ({ - sat_rarity: k[0], - count: k[1], - })); - const recursionInsert = Array.from(recursion.entries()).map(k => ({ - recursive: k[0], - count: k[1], - })); - const typeInsert = Array.from(typeMap.entries()).map(k => ({ - type: k[0], - count: k[1], - })); - // `counts_by_address` and `counts_by_genesis_address` count increases are handled in - // `applyLocations`. - await this.sql` - WITH increase_mime_type AS ( - INSERT INTO counts_by_mime_type ${this.sql(mimeTypeInsert)} + async applyCounts(sql: PgSqlClient, cache: BlockCache) { + if (cache.mimeTypeCounts.size) { + const entries = []; + for (const [mime_type, count] of cache.mimeTypeCounts) entries.push({ mime_type, count }); + await sql` + INSERT INTO counts_by_mime_type ${sql(entries)} ON CONFLICT (mime_type) DO UPDATE SET count = counts_by_mime_type.count + EXCLUDED.count - ), - increase_rarity AS ( - INSERT INTO counts_by_sat_rarity ${this.sql(rarityInsert)} + `; + } + if (cache.satRarityCounts.size) { + const entries = []; + for (const [sat_rarity, count] of cache.satRarityCounts) entries.push({ sat_rarity, count }); + await sql` + INSERT INTO counts_by_sat_rarity ${sql(entries)} ON CONFLICT (sat_rarity) DO UPDATE SET count = counts_by_sat_rarity.count + EXCLUDED.count - ), - increase_recursive AS ( - INSERT INTO counts_by_recursive ${this.sql(recursionInsert)} + `; + } + if (cache.inscriptionTypeCounts.size) { + const entries = []; + for (const [type, count] of cache.inscriptionTypeCounts) entries.push({ type, count }); + await sql` + INSERT INTO counts_by_type ${sql(entries)} + ON CONFLICT (type) DO UPDATE SET count = counts_by_type.count + EXCLUDED.count + `; + } + if (cache.recursiveCounts.size) { + const entries = []; + for (const [recursive, count] of cache.recursiveCounts) entries.push({ recursive, count }); + await sql` + INSERT INTO counts_by_recursive ${sql(entries)} ON CONFLICT (recursive) DO UPDATE SET count = counts_by_recursive.count + EXCLUDED.count - ) - INSERT INTO counts_by_type ${this.sql(typeInsert)} - ON CONFLICT (type) DO UPDATE SET count = counts_by_type.count + EXCLUDED.count - `; - } - - async rollBackInscription(args: { - inscription: InscriptionData; - location: RevealLocationData; - }): Promise { - await this.sql` - WITH decrease_mime_type AS ( - UPDATE counts_by_mime_type SET count = count - 1 - WHERE mime_type = ${args.inscription.mime_type} - ), - decrease_rarity AS ( - UPDATE counts_by_sat_rarity SET count = count - 1 - WHERE sat_rarity = ${args.inscription.sat_rarity} - ), - decrease_recursive AS ( - UPDATE counts_by_recursive SET count = count - 1 - WHERE recursive = ${args.inscription.recursive} - ), - decrease_type AS ( - UPDATE counts_by_type SET count = count - 1 WHERE type = ${ - args.inscription.number < 0 ? DbInscriptionType.cursed : DbInscriptionType.blessed - } - ), - decrease_genesis AS ( - UPDATE counts_by_genesis_address SET count = count - 1 - WHERE address = ${args.location.address} - ) - UPDATE counts_by_address SET count = count - 1 WHERE address = ${args.location.address} - `; + `; + } + if (cache.genesisAddressCounts.size) { + const entries = []; + for (const [address, count] of cache.genesisAddressCounts) entries.push({ address, count }); + await sql` + INSERT INTO counts_by_genesis_address ${sql(entries)} + ON CONFLICT (address) DO UPDATE SET count = counts_by_genesis_address.count + EXCLUDED.count + `; + } + if (cache.inscriptions.length) + await sql` + WITH prev_entry AS ( + SELECT inscription_count_accum + FROM counts_by_block + WHERE block_height < ${cache.blockHeight} + ORDER BY block_height DESC + LIMIT 1 + ) + INSERT INTO counts_by_block + (block_height, block_hash, inscription_count, inscription_count_accum, timestamp) + VALUES ( + ${cache.blockHeight}, ${cache.blockHash}, ${cache.inscriptions.length}, + COALESCE((SELECT inscription_count_accum FROM prev_entry), 0) + ${cache.inscriptions.length}, + TO_TIMESTAMP(${cache.timestamp}) + ) + `; + // Address ownership count is handled in `PgStore`. } - async applyLocations( - writes: { old_address: string | null; new_address: string | null }[], - genesis: boolean = true - ): Promise { - if (writes.length === 0) return; - await this.sqlWriteTransaction(async sql => { - const table = genesis ? sql`counts_by_genesis_address` : sql`counts_by_address`; - const oldAddr = new Map(); - const newAddr = new Map(); - for (const i of writes) { - if (i.old_address) oldAddr.set(i.old_address, (oldAddr.get(i.old_address) ?? 0) + 1); - if (i.new_address) newAddr.set(i.new_address, (newAddr.get(i.new_address) ?? 0) + 1); - } - const oldAddrInsert = Array.from(oldAddr.entries()).map(k => ({ - address: k[0], - count: k[1], - })); - const newAddrInsert = Array.from(newAddr.entries()).map(k => ({ - address: k[0], - count: k[1], - })); - if (oldAddrInsert.length > 0) + async rollBackCounts(sql: PgSqlClient, cache: BlockCache) { + if (cache.inscriptions.length) + await sql`DELETE FROM counts_by_block WHERE block_height = ${cache.blockHeight}`; + if (cache.genesisAddressCounts.size) + for (const [address, count] of cache.genesisAddressCounts) await sql` - INSERT INTO ${table} ${sql(oldAddrInsert)} - ON CONFLICT (address) DO UPDATE SET count = ${table}.count - EXCLUDED.count + UPDATE counts_by_genesis_address SET count = count - ${count} WHERE address = ${address} `; - if (newAddrInsert.length > 0) + if (cache.recursiveCounts.size) + for (const [recursive, count] of cache.recursiveCounts) await sql` - INSERT INTO ${table} ${sql(newAddrInsert)} - ON CONFLICT (address) DO UPDATE SET count = ${table}.count + EXCLUDED.count + UPDATE counts_by_recursive SET count = count - ${count} WHERE recursive = ${recursive} `; - }); - } - - async rollBackCurrentLocation(args: { - curr: DbLocationPointer; - prev: DbLocationPointer; - }): Promise { - await this.sqlWriteTransaction(async sql => { - if (args.curr.address) { + if (cache.inscriptionTypeCounts.size) + for (const [type, count] of cache.inscriptionTypeCounts) await sql` - UPDATE counts_by_address SET count = count - 1 WHERE address = ${args.curr.address} + UPDATE counts_by_type SET count = count - ${count} WHERE type = ${type} `; - } - if (args.prev.address) { + if (cache.satRarityCounts.size) + for (const [sat_rarity, count] of cache.satRarityCounts) await sql` - UPDATE counts_by_address SET count = count + 1 WHERE address = ${args.prev.address} + UPDATE counts_by_sat_rarity SET count = count - ${count} WHERE sat_rarity = ${sat_rarity} `; - } - }); + if (cache.mimeTypeCounts.size) + for (const [mime_type, count] of cache.mimeTypeCounts) + await sql` + UPDATE counts_by_mime_type SET count = count - ${count} WHERE mime_type = ${mime_type} + `; + // Address ownership count is handled in `PgStore`. + } + + async getInscriptionCountPerBlock( + filters: DbInscriptionCountPerBlockFilters + ): Promise { + const fromCondition = filters.from_block_height + ? this.sql`block_height >= ${filters.from_block_height}` + : this.sql``; + + const toCondition = filters.to_block_height + ? this.sql`block_height <= ${filters.to_block_height}` + : this.sql``; + + const where = + filters.from_block_height && filters.to_block_height + ? this.sql`WHERE ${fromCondition} AND ${toCondition}` + : this.sql`WHERE ${fromCondition}${toCondition}`; + + return await this.sql` + SELECT * + FROM counts_by_block + ${filters.from_block_height || filters.to_block_height ? where : this.sql``} + ORDER BY block_height DESC + LIMIT 5000 + `; // roughly 35 days of blocks, assuming 10 minute block times on a full database } private async getBlockCount(from?: number, to?: number): Promise { if (from === undefined && to === undefined) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(inscription_count), 0) AS count - FROM inscriptions_per_block + SELECT COALESCE(SUM(inscription_count), 0)::int AS count + FROM counts_by_block WHERE TRUE ${from !== undefined ? this.sql`AND block_height >= ${from}` : this.sql``} ${to !== undefined ? this.sql`AND block_height <= ${to}` : this.sql``} @@ -201,8 +186,8 @@ export class CountsPgStore extends BasePgStoreModule { private async getBlockHashCount(hash?: string): Promise { if (!hash) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(inscription_count), 0) AS count - FROM inscriptions_per_block + SELECT COALESCE(SUM(inscription_count), 0)::int AS count + FROM counts_by_block WHERE block_hash = ${hash} `; return result[0].count; @@ -212,7 +197,7 @@ export class CountsPgStore extends BasePgStoreModule { const types = type !== undefined ? [type] : [DbInscriptionType.blessed, DbInscriptionType.cursed]; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_type WHERE type IN ${this.sql(types)} `; @@ -222,7 +207,7 @@ export class CountsPgStore extends BasePgStoreModule { private async getMimeTypeCount(mimeType?: string[]): Promise { if (!mimeType) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_mime_type WHERE mime_type IN ${this.sql(mimeType)} `; @@ -232,7 +217,7 @@ export class CountsPgStore extends BasePgStoreModule { private async getSatRarityCount(satRarity?: SatoshiRarity[]): Promise { if (!satRarity) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_sat_rarity WHERE sat_rarity IN ${this.sql(satRarity)} `; @@ -242,17 +227,17 @@ export class CountsPgStore extends BasePgStoreModule { private async getRecursiveCount(recursive?: boolean): Promise { const rec = recursive !== undefined ? [recursive] : [true, false]; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_recursive WHERE recursive IN ${this.sql(rec)} `; return result[0].count; } - private async getAddressCount(address?: string[]): Promise { + async getAddressCount(address?: string[]): Promise { if (!address) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_address WHERE address IN ${this.sql(address)} `; @@ -262,7 +247,7 @@ export class CountsPgStore extends BasePgStoreModule { private async getGenesisAddressCount(genesisAddress?: string[]): Promise { if (!genesisAddress) return 0; const result = await this.sql<{ count: number }[]>` - SELECT COALESCE(SUM(count), 0) AS count + SELECT COALESCE(SUM(count), 0)::int AS count FROM counts_by_genesis_address WHERE address IN ${this.sql(genesisAddress)} `; diff --git a/src/pg/helpers.ts b/src/pg/helpers.ts index 033a1ece..9b399934 100644 --- a/src/pg/helpers.ts +++ b/src/pg/helpers.ts @@ -1,17 +1,7 @@ -import { PgBytea, logger, toEnumValue } from '@hirosystems/api-toolkit'; -import { hexToBuffer, normalizedHexString, parseSatPoint } from '../api/util/helpers'; -import { - BitcoinEvent, - BitcoinInscriptionRevealed, - BitcoinInscriptionTransferred, -} from '@hirosystems/chainhook-client'; -import { - DbLocationTransferType, - InscriptionEventData, - InscriptionTransferData, - InscriptionRevealData, -} from './types'; -import { OrdinalSatoshi } from '../api/util/ordinal-satoshi'; +import { PgBytea, logger } from '@hirosystems/api-toolkit'; +import { hexToBuffer } from '../api/util/helpers'; +import { BitcoinInscriptionRevealed } from '@hirosystems/chainhook-client'; +import { DbLocationTransferType } from './types'; /** * Returns a list of referenced inscription ids from inscription content. @@ -28,28 +18,6 @@ export function getInscriptionRecursion(content: PgBytea): string[] { return result; } -/** - * Returns the values from settled Promise results. - * Throws if any Promise is rejected. - * This can be used with Promise.allSettled to get the values from all promises, - * instead of Promise.all which will swallow following unhandled rejections. - * @param settles - Array of `Promise.allSettled()` results - * @returns Array of Promise result values - */ -export function throwOnFirstRejected(settles: { - [K in keyof T]: PromiseSettledResult; -}): T { - const values: T = [] as any; - for (const promise of settles) { - if (promise.status === 'rejected') throw promise.reason; - - // Note: Pushing to result `values` array is required for type inference - // Compared to e.g. `settles.map(s => s.value)` - values.push(promise.value); - } - return values; -} - export function objRemoveUndefinedValues(obj: object) { Object.keys(obj).forEach(key => (obj as any)[key] === undefined && delete (obj as any)[key]); } @@ -63,23 +31,13 @@ export function removeNullBytes(input: string): string { return input.replace(/\x00/g, ''); } -function updateFromOrdhookInscriptionRevealed(args: { - block_height: number; - block_hash: string; - tx_id: string; - timestamp: number; - reveal: BitcoinInscriptionRevealed; -}): InscriptionRevealData { - const satoshi = new OrdinalSatoshi(args.reveal.ordinal_number); - const satpoint = parseSatPoint(args.reveal.satpoint_post_inscription); - const recursive_refs = getInscriptionRecursion(args.reveal.content_bytes); - const content_type = removeNullBytes(args.reveal.content_type); +export function getTransferType(reveal: BitcoinInscriptionRevealed) { let transfer_type = DbLocationTransferType.transferred; - if (args.reveal.inscriber_address == null || args.reveal.inscriber_address == '') { - if (args.reveal.inscription_output_value == 0) { - if (args.reveal.inscription_pointer !== 0 && args.reveal.inscription_pointer !== null) { + if (reveal.inscriber_address == null || reveal.inscriber_address == '') { + if (reveal.inscription_output_value == 0) { + if (reveal.inscription_pointer !== 0 && reveal.inscription_pointer !== null) { logger.warn( - `Detected inscription reveal with no address and no output value but a valid pointer ${args.reveal.inscription_id}` + `Detected inscription reveal with no address and no output value but a valid pointer ${reveal.inscription_id}` ); } transfer_type = DbLocationTransferType.spentInFees; @@ -87,109 +45,5 @@ function updateFromOrdhookInscriptionRevealed(args: { transfer_type = DbLocationTransferType.burnt; } } - return { - inscription: { - genesis_id: args.reveal.inscription_id, - mime_type: content_type.split(';')[0], - content_type, - content_length: args.reveal.content_length, - number: args.reveal.inscription_number.jubilee, - classic_number: args.reveal.inscription_number.classic, - content: removeNullBytes(args.reveal.content_bytes), - fee: args.reveal.inscription_fee.toString(), - curse_type: args.reveal.curse_type ? JSON.stringify(args.reveal.curse_type) : null, - sat_ordinal: args.reveal.ordinal_number.toString(), - sat_rarity: satoshi.rarity, - sat_coinbase_height: satoshi.blockHeight, - recursive: recursive_refs.length > 0, - metadata: args.reveal.metadata ? JSON.stringify(args.reveal.metadata) : null, - parent: args.reveal.parent, - }, - location: { - block_hash: args.block_hash, - block_height: args.block_height, - tx_id: args.tx_id, - tx_index: args.reveal.tx_index, - block_transfer_index: null, - genesis_id: args.reveal.inscription_id, - address: args.reveal.inscriber_address, - output: `${satpoint.tx_id}:${satpoint.vout}`, - offset: satpoint.offset ?? null, - prev_output: null, - prev_offset: null, - value: args.reveal.inscription_output_value.toString(), - timestamp: args.timestamp, - transfer_type, - }, - recursive_refs, - }; -} - -function updateFromOrdhookInscriptionTransferred(args: { - block_height: number; - block_hash: string; - tx_id: string; - timestamp: number; - blockTransferIndex: number; - transfer: BitcoinInscriptionTransferred; -}): InscriptionTransferData { - const satpoint = parseSatPoint(args.transfer.satpoint_post_transfer); - const prevSatpoint = parseSatPoint(args.transfer.satpoint_pre_transfer); - return { - location: { - block_hash: args.block_hash, - block_height: args.block_height, - tx_id: args.tx_id, - tx_index: args.transfer.tx_index, - block_transfer_index: args.blockTransferIndex, - ordinal_number: args.transfer.ordinal_number.toString(), - address: args.transfer.destination.value ?? null, - output: `${satpoint.tx_id}:${satpoint.vout}`, - offset: satpoint.offset ?? null, - prev_output: `${prevSatpoint.tx_id}:${prevSatpoint.vout}`, - prev_offset: prevSatpoint.offset ?? null, - value: args.transfer.post_transfer_output_value - ? args.transfer.post_transfer_output_value.toString() - : null, - timestamp: args.timestamp, - transfer_type: - toEnumValue(DbLocationTransferType, args.transfer.destination.type) ?? - DbLocationTransferType.transferred, - }, - }; -} - -export function revealInsertsFromOrdhookEvent(event: BitcoinEvent): InscriptionEventData[] { - // Keep the relative ordering of a transfer within a block for faster future reads. - let blockTransferIndex = 0; - const block_height = event.block_identifier.index; - const block_hash = normalizedHexString(event.block_identifier.hash); - const writes: InscriptionEventData[] = []; - for (const tx of event.transactions) { - const tx_id = normalizedHexString(tx.transaction_identifier.hash); - for (const operation of tx.metadata.ordinal_operations) { - if (operation.inscription_revealed) - writes.push( - updateFromOrdhookInscriptionRevealed({ - block_hash, - block_height, - tx_id, - timestamp: event.timestamp, - reveal: operation.inscription_revealed, - }) - ); - if (operation.inscription_transferred) - writes.push( - updateFromOrdhookInscriptionTransferred({ - block_hash, - block_height, - tx_id, - timestamp: event.timestamp, - blockTransferIndex: blockTransferIndex++, - transfer: operation.inscription_transferred, - }) - ); - } - } - return writes; + return transfer_type; } diff --git a/src/pg/pg-store.ts b/src/pg/pg-store.ts index 433cf3b3..0c9267b4 100644 --- a/src/pg/pg-store.ts +++ b/src/pg/pg-store.ts @@ -2,14 +2,13 @@ import { BasePgStore, PgConnectionVars, PgSqlClient, - PgSqlQuery, batchIterate, connectPostgres, logger, runMigrations, stopwatch, } from '@hirosystems/api-toolkit'; -import { BitcoinEvent, Payload } from '@hirosystems/chainhook-client'; +import { BitcoinEvent, BitcoinPayload } from '@hirosystems/chainhook-client'; import * as path from 'path'; import * as postgres from 'postgres'; import { Order, OrderBy } from '../api/schemas'; @@ -17,29 +16,21 @@ import { ENV } from '../env'; import { Brc20PgStore } from './brc20/brc20-pg-store'; import { CountsPgStore } from './counts/counts-pg-store'; import { getIndexResultCountType } from './counts/helpers'; -import { revealInsertsFromOrdhookEvent } from './helpers'; import { DbFullyLocatedInscriptionResult, DbInscriptionContent, - DbInscriptionCountPerBlock, - DbInscriptionCountPerBlockFilters, DbInscriptionIndexFilters, DbInscriptionIndexOrder, DbInscriptionIndexPaging, DbInscriptionLocationChange, DbLocation, - DbLocationPointer, - DbLocationPointerInsert, DbPaginatedResult, - InscriptionEventData, - LOCATIONS_COLUMNS, - InscriptionInsert, - LocationInsert, - LocationData, } from './types'; +import { normalizedHexString } from '../api/util/helpers'; +import { BlockCache } from './block-cache'; export const MIGRATIONS_DIR = path.join(__dirname, '../../migrations'); -export const ORDINALS_GENESIS_BLOCK = 767430; +const ORDINALS_GENESIS_BLOCK = 767430; export const INSERT_BATCH_SIZE = 4000; type InscriptionIdentifier = { genesis_id: string } | { number: number }; @@ -83,62 +74,293 @@ export class PgStore extends BasePgStore { * chain re-orgs. * @param args - Apply/Rollback Ordhook events */ - async updateInscriptions(payload: Payload): Promise { - let updatedBlockHeightMin = Infinity; + async updateInscriptions(payload: BitcoinPayload): Promise { await this.sqlWriteTransaction(async sql => { - // ROLLBACK - for (const rollbackEvent of payload.rollback) { - const event = rollbackEvent as BitcoinEvent; - logger.info(`PgStore rolling back block ${event.block_identifier.index}`); + const streamed = payload.chainhook.is_streaming_blocks; + for (const event of payload.rollback) { + logger.info(`PgStore rollback block ${event.block_identifier.index}`); const time = stopwatch(); - const rollbacks = revealInsertsFromOrdhookEvent(event); - await this.brc20.updateBrc20Operations(event, 'rollback'); - for (const writeChunk of batchIterate(rollbacks, 1000)) - await this.rollBackInscriptions(writeChunk); - updatedBlockHeightMin = Math.min(updatedBlockHeightMin, event.block_identifier.index); + await this.updateInscriptionsEvent(sql, event, 'rollback', streamed); + await this.brc20.updateBrc20Operations(sql, event, 'rollback'); + await this.updateChainTipBlockHeight(sql, event.block_identifier.index - 1); logger.info( - `PgStore rolled back block ${ + `PgStore rollback block ${ event.block_identifier.index - } in ${time.getElapsedSeconds()}s` + } finished in ${time.getElapsedSeconds()}s` ); - await this.updateChainTipBlockHeight(event.block_identifier.index - 1); } - - // APPLY - for (const applyEvent of payload.apply) { - // Check where we're at in terms of ingestion, e.g. block height and max blessed inscription - // number. This will let us determine if we should skip ingesting this block or throw an - // error if a gap is detected. - const currentBlockHeight = await this.getChainTipBlockHeight(); - const event = applyEvent as BitcoinEvent; - if ( - event.block_identifier.index <= currentBlockHeight && - event.block_identifier.index !== ORDINALS_GENESIS_BLOCK - ) { - logger.info( - `PgStore skipping ingestion for previously seen block ${event.block_identifier.index}, current chain tip is at ${currentBlockHeight}` - ); + for (const event of payload.apply) { + if (await this.isBlockIngested(event)) { + logger.warn(`PgStore skipping previously seen block ${event.block_identifier.index}`); continue; } - logger.info(`PgStore ingesting block ${event.block_identifier.index}`); + logger.info(`PgStore apply block ${event.block_identifier.index}`); const time = stopwatch(); - const writes = revealInsertsFromOrdhookEvent(event); - for (const writeChunk of batchIterate(writes, INSERT_BATCH_SIZE)) - await this.insertInscriptions(writeChunk, payload.chainhook.is_streaming_blocks); - updatedBlockHeightMin = Math.min(updatedBlockHeightMin, event.block_identifier.index); - await this.brc20.updateBrc20Operations(event, 'apply'); + await this.updateInscriptionsEvent(sql, event, 'apply', streamed); + await this.brc20.updateBrc20Operations(sql, event, 'apply'); + await this.updateChainTipBlockHeight(sql, event.block_identifier.index); logger.info( - `PgStore ingested block ${event.block_identifier.index} in ${time.getElapsedSeconds()}s` + `PgStore apply block ${ + event.block_identifier.index + } finished in ${time.getElapsedSeconds()}s` ); - await this.updateChainTipBlockHeight(event.block_identifier.index); } }); - if (updatedBlockHeightMin !== Infinity) - await this.normalizeInscriptionCount({ min_block_height: updatedBlockHeightMin }); } - private async updateChainTipBlockHeight(block_height: number): Promise { - await this.sql`UPDATE chain_tip SET block_height = ${block_height}`; + private async updateInscriptionsEvent( + sql: PgSqlClient, + event: BitcoinEvent, + direction: 'apply' | 'rollback', + streamed: boolean = false + ) { + const cache = new BlockCache( + event.block_identifier.index, + normalizedHexString(event.block_identifier.hash), + event.timestamp + ); + for (const tx of event.transactions) { + const tx_id = normalizedHexString(tx.transaction_identifier.hash); + for (const operation of tx.metadata.ordinal_operations) { + if (operation.inscription_revealed) { + cache.reveal(operation.inscription_revealed, tx_id); + logger.info( + `PgStore ${direction} reveal inscription #${operation.inscription_revealed.inscription_number.jubilee} (${operation.inscription_revealed.inscription_id}) at block ${cache.blockHeight}` + ); + } + if (operation.inscription_transferred) { + cache.transfer(operation.inscription_transferred, tx_id); + logger.info( + `PgStore ${direction} transfer satoshi ${operation.inscription_transferred.ordinal_number} to ${operation.inscription_transferred.destination.value} at block ${cache.blockHeight}` + ); + } + } + } + switch (direction) { + case 'apply': + await this.applyInscriptions(sql, cache, streamed); + break; + case 'rollback': + await this.rollBackInscriptions(sql, cache, streamed); + break; + } + } + + private async applyInscriptions( + sql: PgSqlClient, + cache: BlockCache, + streamed: boolean + ): Promise { + if (cache.satoshis.length) + for await (const batch of batchIterate(cache.satoshis, INSERT_BATCH_SIZE)) + await sql` + INSERT INTO satoshis ${sql(batch)} + ON CONFLICT (ordinal_number) DO NOTHING + `; + if (cache.inscriptions.length) { + const entries = cache.inscriptions.map(i => ({ + ...i, + timestamp: sql`TO_TIMESTAMP(${i.timestamp})`, + })); + for await (const batch of batchIterate(entries, INSERT_BATCH_SIZE)) + await sql` + INSERT INTO inscriptions ${sql(batch)} + ON CONFLICT (genesis_id) DO NOTHING + `; + } + if (cache.locations.length) { + const entries = cache.locations.map(l => ({ + ...l, + timestamp: sql`TO_TIMESTAMP(${l.timestamp})`, + })); + for await (const batch of batchIterate(entries, INSERT_BATCH_SIZE)) + await sql` + INSERT INTO locations ${sql(batch)} + ON CONFLICT (ordinal_number, block_height, tx_index) DO NOTHING + `; + // Insert block transfers. + let block_transfer_index = 0; + const transferEntries = []; + for (const transfer of cache.locations) { + const transferred = await sql<{ genesis_id: string; number: string }[]>` + SELECT genesis_id, number FROM inscriptions + WHERE ordinal_number = ${transfer.ordinal_number} AND ( + block_height < ${transfer.block_height} + OR (block_height = ${transfer.block_height} AND tx_index < ${transfer.tx_index}) + ) + `; + for (const inscription of transferred) + transferEntries.push({ + genesis_id: inscription.genesis_id, + number: inscription.number, + ordinal_number: transfer.ordinal_number, + block_height: transfer.block_height, + block_hash: transfer.block_hash, + tx_index: transfer.tx_index, + block_transfer_index: block_transfer_index++, + }); + } + for await (const batch of batchIterate(transferEntries, INSERT_BATCH_SIZE)) + await sql` + INSERT INTO inscription_transfers ${sql(batch)} + ON CONFLICT (block_height, block_transfer_index) DO NOTHING + `; + } + if (cache.recursiveRefs.size) + for (const [genesis_id, refs] of cache.recursiveRefs) { + const entries = refs.map(r => ({ genesis_id, ref_genesis_id: r })); + await sql` + INSERT INTO inscription_recursions ${sql(entries)} + ON CONFLICT (genesis_id, ref_genesis_id) DO NOTHING + `; + } + if (cache.currentLocations.size) { + // Deduct counts from previous owners + const moved_sats = [...cache.currentLocations.keys()]; + const prevOwners = await sql<{ address: string; count: number }[]>` + SELECT address, COUNT(*) AS count + FROM current_locations + WHERE ordinal_number IN ${sql(moved_sats)} + GROUP BY address + `; + for (const owner of prevOwners) + await sql` + UPDATE counts_by_address + SET count = count - ${owner.count} + WHERE address = ${owner.address} + `; + // Insert locations + const entries = [...cache.currentLocations.values()]; + for await (const batch of batchIterate(entries, INSERT_BATCH_SIZE)) + await sql` + INSERT INTO current_locations ${sql(batch)} + ON CONFLICT (ordinal_number) DO UPDATE SET + block_height = EXCLUDED.block_height, + tx_index = EXCLUDED.tx_index, + address = EXCLUDED.address + WHERE + EXCLUDED.block_height > current_locations.block_height OR + (EXCLUDED.block_height = current_locations.block_height AND + EXCLUDED.tx_index > current_locations.tx_index) + `; + // Update owner counts + await sql` + WITH new_owners AS ( + SELECT address, COUNT(*) AS count + FROM current_locations + WHERE ordinal_number IN ${sql(moved_sats)} + GROUP BY address + ) + INSERT INTO counts_by_address (address, count) + (SELECT address, count FROM new_owners) + ON CONFLICT (address) DO UPDATE SET count = counts_by_address.count + EXCLUDED.count + `; + if (streamed) + for await (const batch of batchIterate(moved_sats, INSERT_BATCH_SIZE)) + await sql` + UPDATE inscriptions + SET updated_at = NOW() + WHERE ordinal_number IN ${sql(batch)} + `; + } + await this.counts.applyCounts(sql, cache); + } + + private async rollBackInscriptions( + sql: PgSqlClient, + cache: BlockCache, + streamed: boolean + ): Promise { + await this.counts.rollBackCounts(sql, cache); + const moved_sats = [...cache.currentLocations.keys()]; + // Delete old current owners first. + if (cache.currentLocations.size) { + const prevOwners = await sql<{ address: string; count: number }[]>` + SELECT address, COUNT(*) AS count + FROM current_locations + WHERE ordinal_number IN ${sql(moved_sats)} + GROUP BY address + `; + for (const owner of prevOwners) + await sql` + UPDATE counts_by_address + SET count = count - ${owner.count} + WHERE address = ${owner.address} + `; + await sql` + DELETE FROM current_locations WHERE ordinal_number IN ${sql(moved_sats)} + `; + } + if (cache.locations.length) + for (const location of cache.locations) + await sql` + DELETE FROM locations + WHERE ordinal_number = ${location.ordinal_number} + AND block_height = ${location.block_height} + AND tx_index = ${location.tx_index} + `; + if (cache.inscriptions.length) + // This will also delete recursive refs. + for (const inscription of cache.inscriptions) + await sql` + DELETE FROM inscriptions WHERE genesis_id = ${inscription.genesis_id} + `; + if (cache.satoshis.length) + for (const satoshi of cache.satoshis) + await sql` + DELETE FROM satoshis + WHERE ordinal_number = ${satoshi.ordinal_number} AND NOT EXISTS ( + SELECT genesis_id FROM inscriptions WHERE ordinal_number = ${satoshi.ordinal_number} + ) + `; + // Recalculate current locations for affected inscriptions. + if (cache.currentLocations.size) { + for (const ordinal_number of moved_sats) { + await sql` + INSERT INTO current_locations (ordinal_number, block_height, tx_index, address) + ( + SELECT ordinal_number, block_height, tx_index, address + FROM locations + WHERE ordinal_number = ${ordinal_number} + ORDER BY block_height DESC, tx_index DESC + LIMIT 1 + ) + `; + } + await sql` + WITH new_owners AS ( + SELECT address, COUNT(*) AS count + FROM current_locations + WHERE ordinal_number IN ${sql(moved_sats)} + GROUP BY address + ) + INSERT INTO counts_by_address (address, count) + (SELECT address, count FROM new_owners) + ON CONFLICT (address) DO UPDATE SET count = counts_by_address.count + EXCLUDED.count + `; + if (streamed) + for await (const batch of batchIterate(moved_sats, INSERT_BATCH_SIZE)) + await sql` + UPDATE inscriptions + SET updated_at = NOW() + WHERE ordinal_number IN ${sql(batch)} + `; + } + } + + private async isBlockIngested(event: BitcoinEvent): Promise { + const currentBlockHeight = await this.getChainTipBlockHeight(); + if ( + event.block_identifier.index <= currentBlockHeight && + event.block_identifier.index !== ORDINALS_GENESIS_BLOCK + ) { + return true; + } + return false; + } + + private async updateChainTipBlockHeight(sql: PgSqlClient, block_height: number): Promise { + await sql`UPDATE chain_tip SET block_height = ${block_height}`; } async getChainTipBlockHeight(): Promise { @@ -174,7 +396,7 @@ export class PgStore extends BasePgStore { async getInscriptionsPerBlockETag(): Promise { const result = await this.sql<{ block_hash: string; inscription_count: string }[]>` SELECT block_hash, inscription_count - FROM inscriptions_per_block + FROM counts_by_block ORDER BY block_height DESC LIMIT 1 `; @@ -223,13 +445,13 @@ export class PgStore extends BasePgStore { let orderBy = sql`i.number ${order}`; switch (sort?.order_by) { case OrderBy.genesis_block_height: - orderBy = sql`gen.block_height ${order}, gen.tx_index ${order}`; + orderBy = sql`i.block_height ${order}, i.tx_index ${order}`; break; case OrderBy.ordinal: - orderBy = sql`i.sat_ordinal ${order}`; + orderBy = sql`i.ordinal_number ${order}`; break; case OrderBy.rarity: - orderBy = sql`ARRAY_POSITION(ARRAY['common','uncommon','rare','epic','legendary','mythic'], i.sat_rarity) ${order}, i.number DESC`; + orderBy = sql`ARRAY_POSITION(ARRAY['common','uncommon','rare','epic','legendary','mythic'], s.rarity) ${order}, i.number DESC`; break; } // This function will generate a query to be used for getting results or total counts. @@ -239,10 +461,10 @@ export class PgStore extends BasePgStore { ) => sql` SELECT ${columns} FROM inscriptions AS i - INNER JOIN current_locations AS cur ON cur.inscription_id = i.id - INNER JOIN locations AS cur_l ON cur_l.id = cur.location_id - INNER JOIN genesis_locations AS gen ON gen.inscription_id = i.id - INNER JOIN locations AS gen_l ON gen_l.id = gen.location_id + INNER JOIN current_locations AS cur ON cur.ordinal_number = i.ordinal_number + INNER JOIN locations AS cur_l ON cur_l.ordinal_number = cur.ordinal_number AND cur_l.block_height = cur.block_height AND cur_l.tx_index = cur.tx_index + INNER JOIN locations AS gen_l ON gen_l.ordinal_number = i.ordinal_number AND gen_l.block_height = i.block_height AND gen_l.tx_index = i.tx_index + INNER JOIN satoshis AS s ON s.ordinal_number = i.ordinal_number WHERE TRUE ${ filters?.genesis_id?.length @@ -251,7 +473,7 @@ export class PgStore extends BasePgStore { } ${ filters?.genesis_block_height - ? sql`AND gen.block_height = ${filters.genesis_block_height}` + ? sql`AND i.block_height = ${filters.genesis_block_height}` : sql`` } ${ @@ -261,40 +483,42 @@ export class PgStore extends BasePgStore { } ${ filters?.from_genesis_block_height - ? sql`AND gen.block_height >= ${filters.from_genesis_block_height}` + ? sql`AND i.block_height >= ${filters.from_genesis_block_height}` : sql`` } ${ filters?.to_genesis_block_height - ? sql`AND gen.block_height <= ${filters.to_genesis_block_height}` + ? sql`AND i.block_height <= ${filters.to_genesis_block_height}` : sql`` } ${ filters?.from_sat_coinbase_height - ? sql`AND i.sat_coinbase_height >= ${filters.from_sat_coinbase_height}` + ? sql`AND s.coinbase_height >= ${filters.from_sat_coinbase_height}` : sql`` } ${ filters?.to_sat_coinbase_height - ? sql`AND i.sat_coinbase_height <= ${filters.to_sat_coinbase_height}` + ? sql`AND s.coinbase_height <= ${filters.to_sat_coinbase_height}` : sql`` } ${ filters?.from_genesis_timestamp - ? sql`AND gen_l.timestamp >= to_timestamp(${filters.from_genesis_timestamp})` + ? sql`AND i.timestamp >= to_timestamp(${filters.from_genesis_timestamp})` : sql`` } ${ filters?.to_genesis_timestamp - ? sql`AND gen_l.timestamp <= to_timestamp(${filters.to_genesis_timestamp})` + ? sql`AND i.timestamp <= to_timestamp(${filters.to_genesis_timestamp})` : sql`` } ${ filters?.from_sat_ordinal - ? sql`AND i.sat_ordinal >= ${filters.from_sat_ordinal}` + ? sql`AND i.ordinal_number >= ${filters.from_sat_ordinal}` : sql`` } - ${filters?.to_sat_ordinal ? sql`AND i.sat_ordinal <= ${filters.to_sat_ordinal}` : sql``} + ${ + filters?.to_sat_ordinal ? sql`AND i.ordinal_number <= ${filters.to_sat_ordinal}` : sql`` + } ${filters?.number?.length ? sql`AND i.number IN ${sql(filters.number)}` : sql``} ${ filters?.from_number !== undefined ? sql`AND i.number >= ${filters.from_number}` : sql`` @@ -303,18 +527,14 @@ export class PgStore extends BasePgStore { ${filters?.address?.length ? sql`AND cur.address IN ${sql(filters.address)}` : sql``} ${filters?.mime_type?.length ? sql`AND i.mime_type IN ${sql(filters.mime_type)}` : sql``} ${filters?.output ? sql`AND cur_l.output = ${filters.output}` : sql``} - ${ - filters?.sat_rarity?.length - ? sql`AND i.sat_rarity IN ${sql(filters.sat_rarity)}` - : sql`` - } - ${filters?.sat_ordinal ? sql`AND i.sat_ordinal = ${filters.sat_ordinal}` : sql``} + ${filters?.sat_rarity?.length ? sql`AND s.rarity IN ${sql(filters.sat_rarity)}` : sql``} + ${filters?.sat_ordinal ? sql`AND i.ordinal_number = ${filters.sat_ordinal}` : sql``} ${filters?.recursive !== undefined ? sql`AND i.recursive = ${filters.recursive}` : sql``} ${filters?.cursed === true ? sql`AND i.number < 0` : sql``} ${filters?.cursed === false ? sql`AND i.number >= 0` : sql``} ${ filters?.genesis_address?.length - ? sql`AND gen.address IN ${sql(filters.genesis_address)}` + ? sql`AND i.address IN ${sql(filters.genesis_address)}` : sql`` } ${sorting} @@ -328,21 +548,20 @@ export class PgStore extends BasePgStore { i.content_length, i.fee AS genesis_fee, i.curse_type, - i.sat_ordinal, - i.sat_rarity, - i.sat_coinbase_height, + i.ordinal_number AS sat_ordinal, + s.rarity AS sat_rarity, + s.coinbase_height AS sat_coinbase_height, i.recursive, ( - SELECT STRING_AGG(ii.genesis_id, ',') + SELECT STRING_AGG(ir.ref_genesis_id, ',') FROM inscription_recursions AS ir - INNER JOIN inscriptions AS ii ON ii.id = ir.ref_inscription_id - WHERE ir.inscription_id = i.id + WHERE ir.genesis_id = i.genesis_id ) AS recursion_refs, - gen.block_height AS genesis_block_height, + i.block_height AS genesis_block_height, gen_l.block_hash AS genesis_block_hash, gen_l.tx_id AS genesis_tx_id, - gen_l.timestamp AS genesis_timestamp, - gen.address AS genesis_address, + i.timestamp AS genesis_timestamp, + i.address AS genesis_address, cur_l.tx_id, cur.address, cur_l.output, @@ -372,18 +591,19 @@ export class PgStore extends BasePgStore { args: InscriptionIdentifier & { limit: number; offset: number } ): Promise> { const results = await this.sql<({ total: number } & DbLocation)[]>` - SELECT ${this.sql(LOCATIONS_COLUMNS)}, COUNT(*) OVER() as total - FROM locations - WHERE genesis_id = ( - SELECT genesis_id FROM inscriptions - WHERE ${ - 'number' in args - ? this.sql`number = ${args.number}` - : this.sql`genesis_id = ${args.genesis_id}` - } - LIMIT 1 - ) - ORDER BY block_height DESC, tx_index DESC + SELECT l.*, COUNT(*) OVER() as total + FROM locations AS l + INNER JOIN inscriptions AS i ON i.ordinal_number = l.ordinal_number + WHERE ${ + 'number' in args + ? this.sql`i.number = ${args.number}` + : this.sql`i.genesis_id = ${args.genesis_id}` + } + AND ( + (l.block_height > i.block_height) + OR (l.block_height = i.block_height AND l.tx_index >= i.tx_index) + ) + ORDER BY l.block_height DESC, l.tx_index DESC LIMIT ${args.limit} OFFSET ${args.offset} `; @@ -397,411 +617,70 @@ export class PgStore extends BasePgStore { args: { block_height?: number; block_hash?: string } & DbInscriptionIndexPaging ): Promise> { const results = await this.sql<({ total: number } & DbInscriptionLocationChange)[]>` - WITH max_transfer_index AS ( - SELECT MAX(block_transfer_index) FROM locations WHERE ${ + WITH transfer_total AS ( + SELECT MAX(block_transfer_index) AS total FROM inscription_transfers WHERE ${ 'block_height' in args ? this.sql`block_height = ${args.block_height}` : this.sql`block_hash = ${args.block_hash}` - } AND block_transfer_index IS NOT NULL + } ), - transfers AS ( + transfer_data AS ( SELECT - i.id AS inscription_id, - i.genesis_id, - i.number, - l.id AS to_id, + t.number, + t.genesis_id, + t.ordinal_number, + t.block_height, + t.tx_index, + t.block_transfer_index, ( - SELECT id - FROM locations AS ll - WHERE - ll.inscription_id = i.id - AND ( - ll.block_height < l.block_height OR - (ll.block_height = l.block_height AND ll.tx_index < l.tx_index) - ) - ORDER BY ll.block_height DESC + SELECT l.block_height || ',' || l.tx_index + FROM locations AS l + WHERE l.ordinal_number = t.ordinal_number AND ( + l.block_height < t.block_height OR + (l.block_height = t.block_height AND l.tx_index < t.tx_index) + ) + ORDER BY l.block_height DESC, l.tx_index DESC LIMIT 1 - ) AS from_id - FROM locations AS l - INNER JOIN inscriptions AS i ON l.inscription_id = i.id + ) AS from_data + FROM inscription_transfers AS t WHERE ${ 'block_height' in args - ? this.sql`l.block_height = ${args.block_height}` - : this.sql`l.block_hash = ${args.block_hash}` + ? this.sql`t.block_height = ${args.block_height}` + : this.sql`t.block_hash = ${args.block_hash}` } - AND l.block_transfer_index IS NOT NULL - AND l.block_transfer_index <= ((SELECT max FROM max_transfer_index) - ${args.offset}::int) - AND l.block_transfer_index > - ((SELECT max FROM max_transfer_index) - (${args.offset}::int + ${args.limit}::int)) + AND t.block_transfer_index <= ((SELECT total FROM transfer_total) - ${args.offset}::int) + AND t.block_transfer_index > + ((SELECT total FROM transfer_total) - (${args.offset}::int + ${args.limit}::int)) ) SELECT - t.genesis_id, - t.number, - (SELECT max FROM max_transfer_index) + 1 AS total, - ${this.sql.unsafe(LOCATIONS_COLUMNS.map(c => `lf.${c} AS from_${c}`).join(','))}, - ${this.sql.unsafe(LOCATIONS_COLUMNS.map(c => `lt.${c} AS to_${c}`).join(','))} - FROM transfers AS t - INNER JOIN locations AS lf ON t.from_id = lf.id - INNER JOIN locations AS lt ON t.to_id = lt.id - ORDER BY lt.block_transfer_index DESC + td.genesis_id, + td.number, + lf.block_height AS from_block_height, + lf.block_hash AS from_block_hash, + lf.tx_id AS from_tx_id, + lf.address AS from_address, + lf.output AS from_output, + lf.offset AS from_offset, + lf.value AS from_value, + lf.timestamp AS from_timestamp, + lt.block_height AS to_block_height, + lt.block_hash AS to_block_hash, + lt.tx_id AS to_tx_id, + lt.address AS to_address, + lt.output AS to_output, + lt.offset AS to_offset, + lt.value AS to_value, + lt.timestamp AS to_timestamp, + (SELECT total FROM transfer_total) + 1 AS total + FROM transfer_data AS td + INNER JOIN locations AS lf ON td.ordinal_number = lf.ordinal_number AND lf.block_height = SPLIT_PART(td.from_data, ',', 1)::int AND lf.tx_index = SPLIT_PART(td.from_data, ',', 2)::int + INNER JOIN locations AS lt ON td.ordinal_number = lt.ordinal_number AND td.block_height = lt.block_height AND td.tx_index = lt.tx_index + ORDER BY td.block_height DESC, td.block_transfer_index DESC `; return { total: results[0]?.total ?? 0, results: results ?? [], }; } - - async getInscriptionCountPerBlock( - filters: DbInscriptionCountPerBlockFilters - ): Promise { - const fromCondition = filters.from_block_height - ? this.sql`block_height >= ${filters.from_block_height}` - : this.sql``; - - const toCondition = filters.to_block_height - ? this.sql`block_height <= ${filters.to_block_height}` - : this.sql``; - - const where = - filters.from_block_height && filters.to_block_height - ? this.sql`WHERE ${fromCondition} AND ${toCondition}` - : this.sql`WHERE ${fromCondition}${toCondition}`; - - return await this.sql` - SELECT * - FROM inscriptions_per_block - ${filters.from_block_height || filters.to_block_height ? where : this.sql``} - ORDER BY block_height DESC - LIMIT 5000 - `; // roughly 35 days of blocks, assuming 10 minute block times on a full database - } - - private async insertInscriptions( - reveals: InscriptionEventData[], - streamed: boolean - ): Promise { - if (reveals.length === 0) return; - await this.sqlWriteTransaction(async sql => { - // 1. Write inscription reveals - const inscriptionInserts: InscriptionInsert[] = []; - for (const r of reveals) if ('inscription' in r) inscriptionInserts.push(r.inscription); - if (inscriptionInserts.length) - await sql` - INSERT INTO inscriptions ${sql(inscriptionInserts)} - ON CONFLICT ON CONSTRAINT inscriptions_number_unique DO UPDATE SET - genesis_id = EXCLUDED.genesis_id, - mime_type = EXCLUDED.mime_type, - content_type = EXCLUDED.content_type, - content_length = EXCLUDED.content_length, - content = EXCLUDED.content, - fee = EXCLUDED.fee, - sat_ordinal = EXCLUDED.sat_ordinal, - sat_rarity = EXCLUDED.sat_rarity, - sat_coinbase_height = EXCLUDED.sat_coinbase_height, - updated_at = NOW() - `; - - // 2. Write locations and transfers - const locationInserts: LocationInsert[] = []; - const revealOutputs: InscriptionEventData[] = []; - const transferredOrdinalNumbersSet = new Set(); - for (const r of reveals) - if ('inscription' in r) { - revealOutputs.push(r); - locationInserts.push({ - ...r.location, - inscription_id: sql`(SELECT id FROM inscriptions WHERE genesis_id = ${r.location.genesis_id})`, - timestamp: sql`TO_TIMESTAMP(${r.location.timestamp})`, - }); - } else { - transferredOrdinalNumbersSet.add(r.location.ordinal_number); - // Transfers can move multiple inscriptions in the same sat, we must expand all of them so - // we can update their respective locations. - // TODO: This could probably be optimized to use fewer queries. - const inscriptionIds = await sql<{ id: string; genesis_id: string }[]>` - SELECT id, genesis_id FROM inscriptions WHERE sat_ordinal = ${r.location.ordinal_number} - `; - for (const row of inscriptionIds) { - revealOutputs.push(r); - locationInserts.push({ - genesis_id: row.genesis_id, - inscription_id: row.id, - block_height: r.location.block_height, - block_hash: r.location.block_hash, - tx_id: r.location.tx_id, - tx_index: r.location.tx_index, - address: r.location.address, - output: r.location.output, - offset: r.location.offset, - prev_output: r.location.prev_output, - prev_offset: r.location.prev_offset, - value: r.location.value, - transfer_type: r.location.transfer_type, - block_transfer_index: r.location.block_transfer_index, - timestamp: sql`TO_TIMESTAMP(${r.location.timestamp})`, - }); - } - } - const pointers: DbLocationPointerInsert[] = []; - for (const batch of batchIterate(locationInserts, INSERT_BATCH_SIZE)) { - const pointerBatch = await sql` - INSERT INTO locations ${sql(batch)} - ON CONFLICT ON CONSTRAINT locations_inscription_id_block_height_tx_index_unique DO UPDATE SET - genesis_id = EXCLUDED.genesis_id, - block_hash = EXCLUDED.block_hash, - tx_id = EXCLUDED.tx_id, - address = EXCLUDED.address, - value = EXCLUDED.value, - output = EXCLUDED.output, - "offset" = EXCLUDED.offset, - timestamp = EXCLUDED.timestamp - RETURNING inscription_id, id AS location_id, block_height, tx_index, address - `; - await this.updateInscriptionLocationPointers(pointerBatch); - pointers.push(...pointerBatch); - } - if (streamed && transferredOrdinalNumbersSet.size) - await sql` - UPDATE inscriptions - SET updated_at = NOW() - WHERE sat_ordinal IN ${sql([...transferredOrdinalNumbersSet])} - `; - - for (const reveal of reveals) { - const action = - 'inscription' in reveal - ? `reveal #${reveal.inscription.number} (${reveal.location.genesis_id})` - : `transfer sat ${reveal.location.ordinal_number}`; - logger.info(`PgStore ${action} at block ${reveal.location.block_height}`); - } - - // 3. Recursions and counts - await this.updateInscriptionRecursions(reveals); - await this.counts.applyInscriptions(inscriptionInserts); - }); - } - - private async normalizeInscriptionCount(args: { min_block_height: number }): Promise { - await this.sqlWriteTransaction(async sql => { - await sql` - DELETE FROM inscriptions_per_block - WHERE block_height >= ${args.min_block_height} - `; - // - gets highest total for a block < min_block_height - // - calculates new totals for all blocks >= min_block_height - // - inserts new totals - await sql` - WITH previous AS ( - SELECT * - FROM inscriptions_per_block - WHERE block_height < ${args.min_block_height} - ORDER BY block_height DESC - LIMIT 1 - ), updated_blocks AS ( - SELECT - l.block_height, - MIN(l.block_hash), - COUNT(*) AS inscription_count, - COALESCE((SELECT previous.inscription_count_accum FROM previous), 0) + (SUM(COUNT(*)) OVER (ORDER BY l.block_height ASC)) AS inscription_count_accum, - MIN(l.timestamp) - FROM locations AS l - INNER JOIN genesis_locations AS g ON g.location_id = l.id - WHERE l.block_height >= ${args.min_block_height} - GROUP BY l.block_height - ORDER BY l.block_height ASC - ) - INSERT INTO inscriptions_per_block - SELECT * FROM updated_blocks - ON CONFLICT (block_height) DO UPDATE SET - block_hash = EXCLUDED.block_hash, - inscription_count = EXCLUDED.inscription_count, - inscription_count_accum = EXCLUDED.inscription_count_accum, - timestamp = EXCLUDED.timestamp; - `; - }); - } - - private async rollBackInscriptions(rollbacks: InscriptionEventData[]): Promise { - if (rollbacks.length === 0) return; - await this.sqlWriteTransaction(async sql => { - // Roll back events in reverse so BRC-20 keeps a sane order. - for (const rollback of rollbacks.reverse()) { - if ('inscription' in rollback) { - await this.counts.rollBackInscription({ - inscription: rollback.inscription, - location: rollback.location, - }); - await sql`DELETE FROM inscriptions WHERE genesis_id = ${rollback.inscription.genesis_id}`; - logger.info( - `PgStore rollback reveal #${rollback.inscription.number} (${rollback.inscription.genesis_id}) at block ${rollback.location.block_height}` - ); - } else { - await this.recalculateCurrentLocationPointerFromLocationRollBack({ - location: rollback.location, - }); - await sql` - DELETE FROM locations - WHERE output = ${rollback.location.output} AND "offset" = ${rollback.location.offset} - `; - logger.info( - `PgStore rollback transfer for sat ${rollback.location.ordinal_number} at block ${rollback.location.block_height}` - ); - } - } - }); - } - - private async updateInscriptionLocationPointers( - pointers: DbLocationPointerInsert[] - ): Promise { - if (pointers.length === 0) return; - - // Filters pointer args so we enter only one new pointer per inscription. - const distinctPointers = ( - cond: (a: DbLocationPointerInsert, b: DbLocationPointerInsert) => boolean - ): DbLocationPointerInsert[] => { - const out = new Map(); - for (const ptr of pointers) { - if (ptr.inscription_id === null) continue; - const current = out.get(ptr.inscription_id); - out.set(ptr.inscription_id, current ? (cond(current, ptr) ? current : ptr) : ptr); - } - return [...out.values()]; - }; - - await this.sqlWriteTransaction(async sql => { - const distinctIds = [ - ...new Set(pointers.map(i => i.inscription_id).filter(v => v !== null)), - ]; - const genesisPtrs = distinctPointers( - (a, b) => - parseInt(a.block_height) < parseInt(b.block_height) || - (parseInt(a.block_height) === parseInt(b.block_height) && - parseInt(a.tx_index) < parseInt(b.tx_index)) - ); - if (genesisPtrs.length) { - const genesis = await sql<{ old_address: string | null; new_address: string | null }[]>` - WITH old_pointers AS ( - SELECT inscription_id, address - FROM genesis_locations - WHERE inscription_id IN ${sql(distinctIds)} - ), - new_pointers AS ( - INSERT INTO genesis_locations ${sql(genesisPtrs)} - ON CONFLICT (inscription_id) DO UPDATE SET - location_id = EXCLUDED.location_id, - block_height = EXCLUDED.block_height, - tx_index = EXCLUDED.tx_index, - address = EXCLUDED.address - WHERE - EXCLUDED.block_height < genesis_locations.block_height OR - (EXCLUDED.block_height = genesis_locations.block_height AND - EXCLUDED.tx_index < genesis_locations.tx_index) - RETURNING inscription_id, address - ) - SELECT n.address AS new_address, o.address AS old_address - FROM new_pointers AS n - LEFT JOIN old_pointers AS o USING (inscription_id) - `; - await this.counts.applyLocations(genesis, true); - } - - const currentPtrs = distinctPointers( - (a, b) => - parseInt(a.block_height) > parseInt(b.block_height) || - (parseInt(a.block_height) === parseInt(b.block_height) && - parseInt(a.tx_index) > parseInt(b.tx_index)) - ); - if (currentPtrs.length) { - const current = await sql<{ old_address: string | null; new_address: string | null }[]>` - WITH old_pointers AS ( - SELECT inscription_id, address - FROM current_locations - WHERE inscription_id IN ${sql(distinctIds)} - ), - new_pointers AS ( - INSERT INTO current_locations ${sql(currentPtrs)} - ON CONFLICT (inscription_id) DO UPDATE SET - location_id = EXCLUDED.location_id, - block_height = EXCLUDED.block_height, - tx_index = EXCLUDED.tx_index, - address = EXCLUDED.address - WHERE - EXCLUDED.block_height > current_locations.block_height OR - (EXCLUDED.block_height = current_locations.block_height AND - EXCLUDED.tx_index > current_locations.tx_index) - RETURNING inscription_id, address - ) - SELECT n.address AS new_address, o.address AS old_address - FROM new_pointers AS n - LEFT JOIN old_pointers AS o USING (inscription_id) - `; - await this.counts.applyLocations(current, false); - } - }); - } - - private async recalculateCurrentLocationPointerFromLocationRollBack(args: { - location: LocationData; - }): Promise { - await this.sqlWriteTransaction(async sql => { - // Is the location we're rolling back *the* current location? - const current = await sql` - SELECT * - FROM current_locations AS c - INNER JOIN locations AS l ON l.id = c.location_id - WHERE l.output = ${args.location.output} AND l."offset" = ${args.location.offset} - `; - if (current.count > 0) { - const update = await sql` - WITH prev AS ( - SELECT id, block_height, tx_index, address - FROM locations - WHERE inscription_id = ${current[0].inscription_id} AND id <> ${current[0].location_id} - ORDER BY block_height DESC, tx_index DESC - LIMIT 1 - ) - UPDATE current_locations AS c SET - location_id = prev.id, - block_height = prev.block_height, - tx_index = prev.tx_index, - address = prev.address - FROM prev - WHERE c.inscription_id = ${current[0].inscription_id} - RETURNING * - `; - await this.counts.rollBackCurrentLocation({ curr: current[0], prev: update[0] }); - } - }); - } - - private async updateInscriptionRecursions(reveals: InscriptionEventData[]): Promise { - if (reveals.length === 0) return; - const inserts: { - inscription_id: PgSqlQuery; - ref_inscription_id: PgSqlQuery; - ref_inscription_genesis_id: string; - }[] = []; - for (const i of reveals) - if ('inscription' in i && i.recursive_refs?.length) { - const refSet = new Set(i.recursive_refs); - for (const ref of refSet) - inserts.push({ - inscription_id: this - .sql`(SELECT id FROM inscriptions WHERE genesis_id = ${i.inscription.genesis_id} LIMIT 1)`, - ref_inscription_id: this - .sql`(SELECT id FROM inscriptions WHERE genesis_id = ${ref} LIMIT 1)`, - ref_inscription_genesis_id: ref, - }); - } - if (inserts.length === 0) return; - await this.sqlWriteTransaction(async sql => { - for (const chunk of batchIterate(inserts, 500)) - await sql` - INSERT INTO inscription_recursions ${sql(chunk)} - ON CONFLICT ON CONSTRAINT inscription_recursions_unique DO NOTHING - `; - }); - } } diff --git a/src/pg/types.ts b/src/pg/types.ts index 46680b91..3a4e86de 100644 --- a/src/pg/types.ts +++ b/src/pg/types.ts @@ -2,35 +2,38 @@ import { PgNumeric, PgBytea, PgSqlQuery } from '@hirosystems/api-toolkit'; import { Order, OrderBy } from '../api/schemas'; import { SatoshiRarity } from '../api/util/ordinal-satoshi'; -/** - * Updates and inserts - */ +export type DbSatoshiInsert = { + ordinal_number: PgNumeric; + rarity: string; + coinbase_height: number; +}; -export type InscriptionData = { +export type DbInscriptionInsert = { genesis_id: string; + ordinal_number: PgNumeric; number: number; classic_number: number; + block_height: number; + tx_index: number; + address: string | null; mime_type: string; content_type: string; content_length: number; content: PgBytea; fee: PgNumeric; curse_type: string | null; - sat_ordinal: PgNumeric; - sat_rarity: string; - sat_coinbase_height: number; recursive: boolean; metadata: string | null; parent: string | null; + timestamp: number; }; -export type InscriptionInsert = InscriptionData; - -type AbstractLocationData = { +export type DbLocationInsert = { + ordinal_number: PgNumeric; block_height: number; block_hash: string; - tx_id: string; tx_index: number; + tx_id: string; address: string | null; output: string; offset: PgNumeric | null; @@ -38,36 +41,31 @@ type AbstractLocationData = { prev_offset: PgNumeric | null; value: PgNumeric | null; transfer_type: DbLocationTransferType; - block_transfer_index: number | null; -}; - -export type RevealLocationData = AbstractLocationData & { genesis_id: string; timestamp: number }; - -export type TransferLocationData = AbstractLocationData & { - ordinal_number: PgNumeric; timestamp: number; }; -export type LocationData = RevealLocationData | TransferLocationData; - -export type LocationInsert = AbstractLocationData & { - timestamp: PgSqlQuery; - genesis_id: string; - inscription_id: PgSqlQuery | string; -}; - -export type InscriptionRevealData = { - inscription: InscriptionData; - recursive_refs: string[]; - location: RevealLocationData; +export type DbCurrentLocationInsert = { + ordinal_number: PgNumeric; + block_height: number; + tx_index: number; + address: string | null; }; -export type InscriptionTransferData = { - location: TransferLocationData; +type AbstractLocationData = { + block_height: number; + block_hash: string; + tx_id: string; + tx_index: number; + address: string | null; + output: string; + offset: PgNumeric | null; + prev_output: string | null; + prev_offset: PgNumeric | null; + value: PgNumeric | null; + transfer_type: DbLocationTransferType; + block_transfer_index: number | null; }; -export type InscriptionEventData = InscriptionRevealData | InscriptionTransferData; - /** * Selects */ @@ -111,8 +109,6 @@ export enum DbLocationTransferType { } export type DbLocation = { - id: string; - inscription_id: string | null; genesis_id: string; block_height: string; block_hash: string; @@ -127,27 +123,9 @@ export type DbLocation = { timestamp: Date; }; -export type DbLocationPointer = { - inscription_id: number; - location_id: number; - block_height: number; - tx_index: number; - address: string | null; -}; - -export type DbLocationPointerInsert = { - inscription_id: string; - location_id: string; - block_height: string; - tx_index: string; - address: string | null; -}; - export type DbInscriptionLocationChange = { genesis_id: string; number: string; - from_id: string; - from_inscription_id: string; from_block_height: string; from_block_hash: string; from_tx_id: string; @@ -156,10 +134,6 @@ export type DbInscriptionLocationChange = { from_offset: string | null; from_value: string | null; from_timestamp: Date; - from_genesis: boolean; - from_current: boolean; - to_id: string; - to_inscription_id: string; to_block_height: string; to_block_hash: string; to_tx_id: string; @@ -168,37 +142,6 @@ export type DbInscriptionLocationChange = { to_offset: string | null; to_value: string | null; to_timestamp: Date; - to_genesis: boolean; - to_current: boolean; -}; - -export const LOCATIONS_COLUMNS = [ - 'id', - 'inscription_id', - 'genesis_id', - 'block_height', - 'block_hash', - 'tx_id', - 'tx_index', - 'address', - 'output', - 'offset', - 'value', - 'timestamp', -]; - -export type DbInscription = { - id: string; - genesis_id: string; - number: string; - mime_type: string; - content_type: string; - content_length: string; - fee: string; - sat_ordinal: string; - sat_rarity: string; - sat_coinbase_height: string; - recursive: boolean; }; export type DbInscriptionContent = { @@ -207,21 +150,6 @@ export type DbInscriptionContent = { content: string; }; -export const INSCRIPTIONS_COLUMNS = [ - 'id', - 'genesis_id', - 'number', - 'mime_type', - 'content_type', - 'content_length', - 'fee', - 'curse_type', - 'sat_ordinal', - 'sat_rarity', - 'sat_coinbase_height', - 'recursive', -]; - export type DbInscriptionIndexPaging = { limit: number; offset: number; diff --git a/tests/api/inscriptions.test.ts b/tests/api/inscriptions.test.ts index a9693510..0ae354e8 100644 --- a/tests/api/inscriptions.test.ts +++ b/tests/api/inscriptions.test.ts @@ -278,6 +278,7 @@ describe('/inscriptions', () => { recursion_refs: [ '9f4a9b73b0713c5da01c0a47f97c6c001af9028d6bdd9e264dfacbc4e6790201i0', 'f351d86c6e6cae3c64e297e7463095732f216875bcc1f3c03f950a492bb25421i0', + 'b4b27b9a15f928b95a8ce4b418946553b7b313a345254cd9b23d79489175fa5ai0', ], }; @@ -298,7 +299,7 @@ describe('/inscriptions', () => { expect(response2.json()).toStrictEqual(expected); }); - test('shows inscription with null genesis address', async () => { + test('shows inscription with empty genesis address', async () => { await db.updateInscriptions( new TestChainhookPayloadBuilder() .apply() @@ -318,7 +319,7 @@ describe('/inscriptions', () => { inscription_fee: 2805, inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', inscription_output_value: 10000, - inscriber_address: null, + inscriber_address: '', ordinal_number: 257418248345364, ordinal_block_height: 51483, ordinal_offset: 0, @@ -337,8 +338,8 @@ describe('/inscriptions', () => { .build() ); const expected = { - address: null, - genesis_address: null, + address: '', + genesis_address: '', genesis_block_hash: '00000000000000000002a90330a99f67e3f01eb2ce070b45930581e82fb7a91d', genesis_block_height: 775617, content_length: 5, @@ -1463,7 +1464,7 @@ describe('/inscriptions', () => { inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', inscription_output_value: 10000, inscriber_address: 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', - ordinal_number: 257418248345364, + ordinal_number: 257418248345365, ordinal_block_height: 650000, ordinal_offset: 0, satpoint_post_inscription: @@ -1502,7 +1503,7 @@ describe('/inscriptions', () => { offset: '0', number: 1, value: '10000', - sat_ordinal: '257418248345364', + sat_ordinal: '257418248345365', tx_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', sat_coinbase_height: 51483, output: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0', @@ -1603,7 +1604,7 @@ describe('/inscriptions', () => { inscription_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', inscription_output_value: 10000, inscriber_address: 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', - ordinal_number: 257418248345364, + ordinal_number: 257418248345365, ordinal_block_height: 650000, ordinal_offset: 0, satpoint_post_inscription: @@ -1676,7 +1677,7 @@ describe('/inscriptions', () => { offset: '0', number: 1, value: '10000', - sat_ordinal: '257418248345364', + sat_ordinal: '257418248345365', tx_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', sat_coinbase_height: 51483, output: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0', diff --git a/tests/api/sats.test.ts b/tests/api/sats.test.ts index 4747237a..7e036772 100644 --- a/tests/api/sats.test.ts +++ b/tests/api/sats.test.ts @@ -81,7 +81,7 @@ describe('/sats', () => { ); }); - test('returns sat with more than 1 cursed inscription', async () => { + test('returns sat with more than 1 inscription', async () => { await db.updateInscriptions( new TestChainhookPayloadBuilder() .apply() @@ -133,6 +133,7 @@ describe('/sats', () => { inscription_id: 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993i0', inscription_output_value: 10000, inscriber_address: 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + // Same sat. This will also create a transfer for the previous inscription. ordinal_number: 257418248345364, ordinal_block_height: 650000, ordinal_offset: 0, @@ -195,22 +196,71 @@ describe('/sats', () => { genesis_timestamp: 1677803510000, genesis_tx_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0', - location: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0:0', + // Re-inscribed sat is moved to the latest inscription's location. + location: 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993:0:0', mime_type: 'image/png', number: -7, offset: '0', - output: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0', + output: 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993:0', sat_coinbase_height: 51483, sat_ordinal: '257418248345364', sat_rarity: 'common', - timestamp: 1677803510000, - tx_id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc', + timestamp: 1676913207000, + tx_id: 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993', value: '10000', curse_type: '"p2wsh"', recursive: false, recursion_refs: null, }, ]); + + // Inscription -7 should have 2 locations, -1 should only have 1. + let transfersResponse = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/-7/transfers', + }); + expect(transfersResponse.statusCode).toBe(200); + let transferJson = transfersResponse.json(); + expect(transferJson.total).toBe(2); + expect(transferJson.results).toHaveLength(2); + expect(transferJson.results[0].location).toBe( + 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993:0:0' + ); + expect(transferJson.results[1].location).toBe( + '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dc:0:0' + ); + + transfersResponse = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/-1/transfers', + }); + expect(transfersResponse.statusCode).toBe(200); + transferJson = transfersResponse.json(); + expect(transferJson.total).toBe(1); + expect(transferJson.results).toHaveLength(1); + expect(transferJson.results[0].location).toBe( + 'b9cd9489fe30b81d007f753663d12766f1368721a87f4c69056c8215caa57993:0:0' + ); + + // Block transfer activity should reflect all true transfers. + transfersResponse = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/transfers?block=775617', + }); + expect(transfersResponse.statusCode).toBe(200); + transferJson = transfersResponse.json(); + expect(transferJson.total).toBe(0); + expect(transferJson.results).toHaveLength(0); + + transfersResponse = await fastify.inject({ + method: 'GET', + url: '/ordinals/v1/inscriptions/transfers?block=775618', + }); + expect(transfersResponse.statusCode).toBe(200); + transferJson = transfersResponse.json(); + expect(transferJson.total).toBe(1); + expect(transferJson.results).toHaveLength(1); + expect(transferJson.results[0].number).toBe(-7); }); test('returns not found on invalid sats', async () => { diff --git a/tests/ordhook/server.test.ts b/tests/ordhook/server.test.ts index e8035854..a09a217c 100644 --- a/tests/ordhook/server.test.ts +++ b/tests/ordhook/server.test.ts @@ -117,6 +117,10 @@ describe('EventServer', () => { expect(inscr.sat_rarity).toBe('common'); expect(inscr.timestamp.toISOString()).toBe('2023-02-20T17:13:27.000Z'); expect(inscr.value).toBe('10000'); + let count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + ]); + expect(count).toBe(1); // Rollback const payload2 = new TestChainhookPayloadBuilder() @@ -142,6 +146,10 @@ describe('EventServer', () => { expect(c1[0].count).toBe(0); const c2 = await db.sql<{ count: number }[]>`SELECT COUNT(*)::int FROM locations`; expect(c2[0].count).toBe(0); + count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + ]); + expect(count).toBe(0); }); test('parses inscription_transferred apply and rollback', async () => { @@ -257,6 +265,14 @@ describe('EventServer', () => { expect(inscr.sat_rarity).toBe('common'); expect(inscr.timestamp.toISOString()).toBe('2023-02-20T17:13:27.000Z'); expect(inscr.value).toBe('10000'); + let count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + ]); + expect(count).toBe(0); + count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf00000', + ]); + expect(count).toBe(1); // Rollback const payload2 = new TestChainhookPayloadBuilder() @@ -283,6 +299,14 @@ describe('EventServer', () => { const c2 = await db.sql<{ count: number }[]>`SELECT COUNT(*)::int FROM locations`; expect(c2[0].count).toBe(1); await expect(db.getChainTipBlockHeight()).resolves.toBe(775617); + count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf0u2td', + ]); + expect(count).toBe(1); + count = await db.counts.getAddressCount([ + 'bc1p3cyx5e2hgh53w7kpxcvm8s4kkega9gv5wfw7c4qxsvxl0u8x834qf00000', + ]); + expect(count).toBe(0); }); test('multiple inscription pointers on the same block are compared correctly', async () => { @@ -485,6 +509,7 @@ describe('EventServer', () => { const status = await db.sql<{ transfer_type: string }[]>` SELECT transfer_type FROM locations + INNER JOIN inscriptions USING (ordinal_number) WHERE genesis_id = '53957f47697096cef4ad24dae6357b3d7ffe1e3eb9216ce0bb01d6b6a2c8cf4ai0' `; expect(status[0].transfer_type).toBe('spent_in_fees'); @@ -539,6 +564,7 @@ describe('EventServer', () => { const status = await db.sql<{ transfer_type: string }[]>` SELECT transfer_type FROM locations + INNER JOIN inscriptions USING (ordinal_number) WHERE genesis_id = '53957f47697096cef4ad24dae6357b3d7ffe1e3eb9216ce0bb01d6b6a2c8cf4ai0' `; expect(status[0].transfer_type).toBe('burnt');