Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: keep block transfer index for faster transfers per block #260

Merged
merged 1 commit into from
Nov 2, 2023
Merged

fix: keep block transfer index for faster transfers per block #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 14 additions & 0 deletions migrations/1698897577725_locations-location-index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/* eslint-disable @typescript-eslint/naming-convention */
import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';

export const shorthands: ColumnDefinitions | undefined = undefined;

export function up(pgm: MigrationBuilder): void {
pgm.addColumn('locations', {
block_transfer_index: {
type: 'int',
},
});
pgm.addIndex('locations', ['block_height', { name: 'block_transfer_index', sort: 'DESC' }]);
pgm.addIndex('locations', ['block_hash', { name: 'block_transfer_index', sort: 'DESC' }]);
}
36 changes: 23 additions & 13 deletions src/pg/pg-store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import {
BasePgStore,
PgConnectionVars,
PgSqlClient,
PgSqlQuery,
connectPostgres,
isTestEnv,
logger,
runMigrations,
} from '@hirosystems/api-toolkit';
Expand Down Expand Up @@ -43,7 +43,6 @@ import { toEnumValue } from '@hirosystems/api-toolkit';
export const MIGRATIONS_DIR = path.join(__dirname, '../../migrations');

type InscriptionIdentifier = { genesis_id: string } | { number: number };
type PgQueryFragment = postgres.PendingQuery<postgres.Row[]>; // TODO: Move to api-toolkit

export class PgStore extends BasePgStore {
readonly brc20: Brc20PgStore;
Expand Down Expand Up @@ -121,6 +120,8 @@ export class PgStore extends BasePgStore {
}
updatedBlockHeightMin = Math.min(updatedBlockHeightMin, event.block_identifier.index);
}

let blockTransferIndex = 0;
for (const applyEvent of payload.apply) {
const event = applyEvent as BitcoinEvent;
const block_height = event.block_identifier.index;
Expand Down Expand Up @@ -165,6 +166,7 @@ export class PgStore extends BasePgStore {
block_height,
tx_id,
tx_index: reveal.tx_index,
block_transfer_index: null,
genesis_id: reveal.inscription_id,
address: reveal.inscriber_address,
output: `${satpoint.tx_id}:${satpoint.vout}`,
Expand Down Expand Up @@ -203,6 +205,7 @@ export class PgStore extends BasePgStore {
block_height,
tx_id,
tx_index: reveal.tx_index,
block_transfer_index: null,
genesis_id: reveal.inscription_id,
address: reveal.inscriber_address,
output: `${satpoint.tx_id}:${satpoint.vout}`,
Expand All @@ -226,6 +229,7 @@ export class PgStore extends BasePgStore {
block_height,
tx_id,
tx_index: transfer.tx_index,
block_transfer_index: blockTransferIndex++,
genesis_id: transfer.inscription_id,
address: transfer.destination.value ?? null,
output: `${satpoint.tx_id}:${satpoint.vout}`,
Expand Down Expand Up @@ -514,7 +518,14 @@ export class PgStore extends BasePgStore {
args: { block_height?: number; block_hash?: string } & DbInscriptionIndexPaging
): Promise<DbPaginatedResult<DbInscriptionLocationChange>> {
const results = await this.sql<({ total: number } & DbInscriptionLocationChange)[]>`
WITH transfers AS (
WITH max_transfer_index AS (
SELECT MAX(block_transfer_index) FROM locations WHERE ${
'block_height' in args
? this.sql`block_height = ${args.block_height}`
: this.sql`block_hash = ${args.block_hash}`
} AND block_transfer_index IS NOT NULL
),
transfers AS (
SELECT
i.id AS inscription_id,
i.genesis_id,
Expand All @@ -531,31 +542,30 @@ export class PgStore extends BasePgStore {
)
ORDER BY ll.block_height DESC
LIMIT 1
) AS from_id,
COUNT(*) OVER() as total
) AS from_id
FROM locations AS l
INNER JOIN inscriptions AS i ON l.inscription_id = i.id
WHERE
NOT EXISTS (SELECT location_id FROM genesis_locations WHERE location_id = l.id)
AND
${
'block_height' in args
? this.sql`l.block_height = ${args.block_height}`
: this.sql`l.block_hash = ${args.block_hash}`
}
LIMIT ${args.limit}
OFFSET ${args.offset}
AND l.block_transfer_index IS NOT NULL
AND l.block_transfer_index <= ((SELECT max FROM max_transfer_index) - ${args.offset}::int)
AND l.block_transfer_index >
((SELECT max FROM max_transfer_index) - (${args.offset}::int + ${args.limit}::int))
)
SELECT
t.genesis_id,
t.number,
t.total,
(SELECT max FROM max_transfer_index) + 1 AS total,
${this.sql.unsafe(LOCATIONS_COLUMNS.map(c => `lf.${c} AS from_${c}`).join(','))},
${this.sql.unsafe(LOCATIONS_COLUMNS.map(c => `lt.${c} AS to_${c}`).join(','))}
FROM transfers AS t
INNER JOIN locations AS lf ON t.from_id = lf.id
INNER JOIN locations AS lt ON t.to_id = lt.id
ORDER BY to_tx_index DESC
ORDER BY lt.block_transfer_index DESC
`;
return {
total: results[0]?.total ?? 0,
Expand Down Expand Up @@ -870,8 +880,8 @@ export class PgStore extends BasePgStore {
private async updateInscriptionRecursions(reveals: DbRevealInsert[]): Promise<void> {
if (reveals.length === 0) return;
const inserts: {
inscription_id: PgQueryFragment;
ref_inscription_id: PgQueryFragment;
inscription_id: PgSqlQuery;
ref_inscription_id: PgSqlQuery;
ref_inscription_genesis_id: string;
}[] = [];
for (const i of reveals)
Expand Down
1 change: 1 addition & 0 deletions src/pg/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export type DbLocationInsert = {
value: PgNumeric | null;
timestamp: number;
transfer_type: DbLocationTransferType;
block_transfer_index: number | null;
};

export type DbLocation = {
Expand Down
75 changes: 75 additions & 0 deletions tests/inscriptions.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1195,6 +1195,7 @@ describe('/inscriptions', () => {
expect(response3.statusCode).toBe(200);
const json3 = response3.json();
expect(json3.total).toBe(2);
expect(json3.results).toHaveLength(2);
expect(json3.results).toStrictEqual([
{
id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0',
Expand Down Expand Up @@ -1249,6 +1250,80 @@ describe('/inscriptions', () => {
},
},
]);

// Test pagination
const response4 = await fastify.inject({
method: 'GET',
url: '/ordinals/v1/inscriptions/transfers?block=775701&limit=1&offset=0',
});
expect(response4.statusCode).toBe(200);
const json4 = response4.json();
expect(json4.total).toBe(2);
expect(json4.results).toHaveLength(1);
expect(json4.results).toStrictEqual([
{
id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0',
number: 0,
from: {
address: 'bc1pkx5me775s748lzchytzdsw4f0lq04wssxnyk27g8fn3gee8zhjjqsn9tfp',
block_hash: '000000000000000000044b12039abd3112963959d9fd7510ac503ea84dc17002',
block_height: 775701,
location: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:0:0',
offset: '0',
output: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:0',
timestamp: 1676913208000,
tx_id: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e',
value: '8000',
},
to: {
address: 'bc1pkx5me775s748lzchytzdsw4f0lq04wssxnyk27g8fn3gee8zhjjqsn9tfp',
block_hash: '000000000000000000044b12039abd3112963959d9fd7510ac503ea84dc17002',
block_height: 775701,
location: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:1:0',
offset: '0',
output: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:1',
timestamp: 1676913208000,
tx_id: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e',
value: '7500',
},
},
]);
const response5 = await fastify.inject({
method: 'GET',
url: '/ordinals/v1/inscriptions/transfers?block=775701&limit=1&offset=1',
});
expect(response5.statusCode).toBe(200);
const json5 = response5.json();
expect(json5.total).toBe(2);
expect(json5.results).toHaveLength(1);
expect(json5.results).toStrictEqual([
{
id: '38c46a8bf7ec90bc7f6b797e7dc84baa97f4e5fd4286b92fe1b50176d03b18dci0',
number: 0,
from: {
address: 'bc1p3xqwzmddceqrd6x9yxplqzkl5vucta2gqm5szpkmpuvcvgs7g8psjf8htd',
block_hash: '00000000000000000002a90330a99f67e3f01eb2ce070b45930581e82fb7bbbb',
block_height: 775700,
location: 'bdda0d240132bab2af7f797d1507beb1acab6ad43e2c0ef7f96291aea5cc3444:0:0',
offset: '0',
output: 'bdda0d240132bab2af7f797d1507beb1acab6ad43e2c0ef7f96291aea5cc3444:0',
timestamp: 1678122360000,
tx_id: 'bdda0d240132bab2af7f797d1507beb1acab6ad43e2c0ef7f96291aea5cc3444',
value: '9000',
},
to: {
address: 'bc1pkx5me775s748lzchytzdsw4f0lq04wssxnyk27g8fn3gee8zhjjqsn9tfp',
block_hash: '000000000000000000044b12039abd3112963959d9fd7510ac503ea84dc17002',
block_height: 775701,
location: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:0:0',
offset: '0',
output: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e:0',
timestamp: 1676913208000,
tx_id: '5cabafe04aaf98b1f325b0c3ffcbff904dbdb6f3d2e9e451102fda36f1056b5e',
value: '8000',
},
},
]);
});
});

Expand Down