Merge pull request #317 from hirosystems/beta

release to master

.github/workflows/ci.yml

@@ -18,15 +18,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Use Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
 
       - name: Cache node modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         env:
           cache-name: cache-node-modules
         with:
@@ -63,17 +63,17 @@ jobs:
       PGPASSWORD: postgres
       PGDATABASE: postgres
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Use Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
 
       - name: Cache node modules
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         env:
           cache-name: cache-node-modules
         with:
@@ -99,7 +99,9 @@ jobs:
         run: npm run test:${{ matrix.suite }} -- --coverage
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Print integration environment logs
         run: cat docker-compose-logs.txt
@@ -115,14 +117,14 @@ jobs:
       - lint
       - test
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }}
           fetch-depth: 0
           persist-credentials: false
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         id: semantic
         # Only run on non-PR events or only PRs that aren't from forks
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
@@ -137,11 +139,11 @@ jobs:
             conventional-changelog-conventionalcommits@6.1.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker Meta
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: |
             hirosystems/${{ github.event.repository.name }}
@@ -152,13 +154,13 @@ jobs:
             type=semver,pattern={{major}}.{{minor}},value=${{ steps.semantic.outputs.new_release_version }},enable=${{ steps.semantic.outputs.new_release_version != '' }}
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build/Tag/Push Image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           tags: ${{ steps.meta.outputs.tags }}

CHANGELOG.md

@@ -1,3 +1,64 @@
+## [3.0.0-beta.2](https://github.com/hirosystems/ordinals-api/compare/v3.0.0-beta.1...v3.0.0-beta.2) (2024-02-21)
+
+
+### Features
+
+* shutdown gracefully after finishing replaying blocks ([#315](https://github.com/hirosystems/ordinals-api/issues/315)) ([72fd3fd](https://github.com/hirosystems/ordinals-api/commit/72fd3fda24919f14218cbbf02a552dfcf4d8ea52))
+
+## [3.0.0-beta.1](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.6...v3.0.0-beta.1) (2024-02-21)
+
+
+### ⚠ BREAKING CHANGES
+
+* rename chainhook env vars to ordhook (#314)
+
+### Bug Fixes
+
+* rename chainhook env vars to ordhook ([#314](https://github.com/hirosystems/ordinals-api/issues/314)) ([ae4ec01](https://github.com/hirosystems/ordinals-api/commit/ae4ec01326779af00b89907751a6946effe60536))
+
+## [2.3.0-beta.6](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.5...v2.3.0-beta.6) (2024-02-20)
+
+
+### Bug Fixes
+
+* remove unused pg indexes ([#311](https://github.com/hirosystems/ordinals-api/issues/311)) ([94d98d4](https://github.com/hirosystems/ordinals-api/commit/94d98d4e945aef28be7ec19fdb07d121c0ba2ea2))
+
+## [2.3.0-beta.5](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.4...v2.3.0-beta.5) (2024-02-19)
+
+
+### Bug Fixes
+
+* only update timestamp when streaming blocks ([#309](https://github.com/hirosystems/ordinals-api/issues/309)) ([2c9ff17](https://github.com/hirosystems/ordinals-api/commit/2c9ff17ebe8805f3de59046b25387b871debee49))
+
+## [2.3.0-beta.4](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.3...v2.3.0-beta.4) (2024-02-17)
+
+
+### Bug Fixes
+
+* standardize insert batch size ([b9b2448](https://github.com/hirosystems/ordinals-api/commit/b9b2448a1fa1bef16f0a1eed80158ca9b8ca9133))
+
+## [2.3.0-beta.3](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.2...v2.3.0-beta.3) (2024-02-17)
+
+
+### Bug Fixes
+
+* batch inscription location updates ([47525c9](https://github.com/hirosystems/ordinals-api/commit/47525c93a0e71bd9a7df2ea1596d7c90f59ccaac))
+
+## [2.3.0-beta.2](https://github.com/hirosystems/ordinals-api/compare/v2.3.0-beta.1...v2.3.0-beta.2) (2024-02-17)
+
+
+### Bug Fixes
+
+* batch location pointer inserts ([#308](https://github.com/hirosystems/ordinals-api/issues/308)) ([33f8cb2](https://github.com/hirosystems/ordinals-api/commit/33f8cb2576115695d8e54fcc989086afef42ddc7))
+
+## [2.3.0-beta.1](https://github.com/hirosystems/ordinals-api/compare/v2.2.0...v2.3.0-beta.1) (2024-02-12)
+
+
+### Features
+
+* add `metadata` and `parent` columns to inscriptions table ([#305](https://github.com/hirosystems/ordinals-api/issues/305)) ([d71e93a](https://github.com/hirosystems/ordinals-api/commit/d71e93a54ec8018c15bbfc2f88d18bab6606949e))
+* add ordhook debug server ([#306](https://github.com/hirosystems/ordinals-api/issues/306)) ([88ad130](https://github.com/hirosystems/ordinals-api/commit/88ad1302924b8da27b8d98caf3d3351149a45f91))
+
 ## [2.2.0](https://github.com/hirosystems/ordinals-api/compare/v2.1.1...v2.2.0) (2024-02-02)
 
 

migrations/1707770109739_metadata-parent.ts

@@ -0,0 +1,15 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.addColumns('inscriptions', {
+    metadata: {
+      type: 'text',
+    },
+    parent: {
+      type: 'text',
+    },
+  });
+}

migrations/1708471015438_remove-unused-indexes.ts

@@ -0,0 +1,36 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.dropIndex('locations', ['prev_output']);
+  pgm.dropIndex('locations', ['address']);
+  pgm.dropIndex('current_locations', ['block_height']);
+  pgm.dropIndex('brc20_mints', ['address']);
+  pgm.dropIndex('brc20_mints', ['block_height']);
+  pgm.dropIndex('brc20_mints', ['brc20_deploy_id']);
+  pgm.dropIndex('brc20_transfers', ['to_address']);
+  pgm.dropIndex('brc20_transfers', ['from_address']);
+  pgm.dropIndex('brc20_transfers', ['brc20_deploy_id']);
+  pgm.dropIndex('brc20_transfers', ['block_height']);
+  pgm.dropIndex('brc20_deploys', ['address']);
+  pgm.dropIndex('brc20_deploys', ['block_height']);
+  pgm.dropIndex('inscription_recursions', ['ref_inscription_genesis_id']);
+}
+
+export function down(pgm: MigrationBuilder): void {
+  pgm.createIndex('locations', ['prev_output']);
+  pgm.createIndex('locations', ['address']);
+  pgm.createIndex('current_locations', ['block_height']);
+  pgm.createIndex('brc20_mints', ['address']);
+  pgm.createIndex('brc20_mints', ['block_height']);
+  pgm.createIndex('brc20_mints', ['brc20_deploy_id']);
+  pgm.createIndex('brc20_transfers', ['to_address']);
+  pgm.createIndex('brc20_transfers', ['from_address']);
+  pgm.createIndex('brc20_transfers', ['brc20_deploy_id']);
+  pgm.createIndex('brc20_transfers', ['block_height']);
+  pgm.createIndex('brc20_deploys', ['address']);
+  pgm.createIndex('brc20_deploys', ['block_height']);
+  pgm.createIndex('inscription_recursions', ['ref_inscription_genesis_id']);
+}

package.json

@@ -9,6 +9,7 @@
     "build": "rimraf ./dist && tsc --project tsconfig.build.json",
     "start": "node dist/src/index.js",
     "start-ts": "ts-node ./src/index.ts",
+    "start:debug-server": "node dist/util/debug-server.js",
     "test": "jest --runInBand",
     "test:brc-20": "npm run test -- ./tests/brc-20/",
     "test:api": "npm run test -- ./tests/api/",
@@ -53,8 +54,8 @@
     "@fastify/multipart": "^7.1.0",
     "@fastify/swagger": "^8.3.1",
     "@fastify/type-provider-typebox": "^3.2.0",
-    "@hirosystems/api-toolkit": "^1.3.1",
-    "@hirosystems/chainhook-client": "^1.6.0",
+    "@hirosystems/api-toolkit": "^1.4.0",
+    "@hirosystems/chainhook-client": "^1.7.0",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/commit-analyzer": "^10.0.4",
     "@semantic-release/git": "^10.0.1",

src/env.ts

@@ -28,19 +28,31 @@ const schema = Type.Object({
   EXTERNAL_HOSTNAME: Type.String({ default: '127.0.0.1' }),
 
   /** Hostname of the ordhook node we'll use to register predicates */
-  CHAINHOOK_NODE_RPC_HOST: Type.String({ default: '127.0.0.1' }),
+  ORDHOOK_NODE_RPC_HOST: Type.String({ default: '127.0.0.1' }),
   /** Control port of the ordhook node */
-  CHAINHOOK_NODE_RPC_PORT: Type.Number({ default: 20456, minimum: 0, maximum: 65535 }),
+  ORDHOOK_NODE_RPC_PORT: Type.Number({ default: 20456, minimum: 0, maximum: 65535 }),
   /**
    * Authorization token that the ordhook node must send with every event to make sure it's
    * coming from the valid instance
    */
-  CHAINHOOK_NODE_AUTH_TOKEN: Type.String(),
+  ORDHOOK_NODE_AUTH_TOKEN: Type.String(),
   /**
    * Register ordhook predicates automatically when the API is first launched. Set this to `false`
    * if you're configuring your predicates manually for any reason.
    */
-  CHAINHOOK_AUTO_PREDICATE_REGISTRATION: Type.Boolean({ default: true }),
+  ORDHOOK_AUTO_PREDICATE_REGISTRATION: Type.Boolean({ default: true }),
+  /**
+   * Ordhook ingestion mode. Controls the API's Ordhook payload ingestion behavior:
+   * * `default`: The API will stay running and will listen for payloads indefinitely
+   * * `replay`: The API will stay running and listening only for payloads marked as "not streaming"
+   *   by Ordhook (historical replays). Once Ordhook starts streaming recent blocks from its chain
+   *   tip, the API will shut down. Recommended for deployments meant to sync the ordinals chain
+   *   from genesis.
+   */
+  ORDHOOK_INGESTION_MODE: Type.Enum(
+    { default: 'default', replay: 'replay' },
+    { default: 'default' }
+  ),
 
   PGHOST: Type.String(),
   PGPORT: Type.Number({ default: 5432, minimum: 0, maximum: 65535 }),

src/ordhook/server.ts

@@ -8,9 +8,9 @@ import {
   ServerOptions,
   ServerPredicate,
 } from '@hirosystems/chainhook-client';
-import { logger } from '@hirosystems/api-toolkit';
+import { logger, shutdown } from '@hirosystems/api-toolkit';
 
-export const ORDHOOK_BASE_PATH = `http://${ENV.CHAINHOOK_NODE_RPC_HOST}:${ENV.CHAINHOOK_NODE_RPC_PORT}`;
+export const ORDHOOK_BASE_PATH = `http://${ENV.ORDHOOK_NODE_RPC_HOST}:${ENV.ORDHOOK_NODE_RPC_PORT}`;
 export const PREDICATE_UUID = randomUUID();
 
 /**
@@ -20,7 +20,7 @@ export const PREDICATE_UUID = randomUUID();
  */
 export async function startOrdhookServer(args: { db: PgStore }): Promise<ChainhookEventObserver> {
   const predicates: ServerPredicate[] = [];
-  if (ENV.CHAINHOOK_AUTO_PREDICATE_REGISTRATION) {
+  if (ENV.ORDHOOK_AUTO_PREDICATE_REGISTRATION) {
     const blockHeight = await args.db.getChainTipBlockHeight();
     logger.info(`Ordinals predicate starting from block ${blockHeight}...`);
     predicates.push({
@@ -43,9 +43,9 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise<Chainho
   const serverOpts: ServerOptions = {
     hostname: ENV.API_HOST,
     port: ENV.EVENT_PORT,
-    auth_token: ENV.CHAINHOOK_NODE_AUTH_TOKEN,
+    auth_token: ENV.ORDHOOK_NODE_AUTH_TOKEN,
     external_base_url: `http://${ENV.EXTERNAL_HOSTNAME}`,
-    wait_for_chainhook_node: ENV.CHAINHOOK_AUTO_PREDICATE_REGISTRATION,
+    wait_for_chainhook_node: ENV.ORDHOOK_AUTO_PREDICATE_REGISTRATION,
     validate_chainhook_payloads: true,
     body_limit: ENV.EVENT_SERVER_BODY_LIMIT,
     node_type: 'ordhook',
@@ -55,7 +55,14 @@ export async function startOrdhookServer(args: { db: PgStore }): Promise<Chainho
   };
   const server = new ChainhookEventObserver(serverOpts, ordhookOpts);
   await server.start(predicates, async (uuid: string, payload: Payload) => {
-    logger.info(`OrdhookServer received payload from predicate ${uuid}`);
+    const streamed = payload.chainhook.is_streaming_blocks;
+    if (ENV.ORDHOOK_INGESTION_MODE === 'replay' && streamed) {
+      logger.info(`OrdhookServer finished replaying blocks, shutting down`);
+      return shutdown();
+    }
+    logger.info(
+      `OrdhookServer received ${streamed ? 'streamed' : 'replay'} payload from predicate ${uuid}`
+    );
     await args.db.updateInscriptions(payload);
   });
   return server;

src/pg/helpers.ts

@@ -114,6 +114,8 @@ function updateFromOrdhookInscriptionRevealed(args: {
       sat_rarity: satoshi.rarity,
       sat_coinbase_height: satoshi.blockHeight,
       recursive: recursive_refs.length > 0,
+      metadata: args.reveal.metadata ? JSON.stringify(args.reveal.metadata) : null,
+      parent: args.reveal.parent,
     },
     location: {
       block_hash: args.block_hash,