diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json index 9bc91d586..4443eeb6f 100644 --- a/packages/common-cosmos/package.json +++ b/packages/common-cosmos/package.json @@ -22,7 +22,7 @@ "@protobufs/google": "^0.0.10", "@protobufs/ibc": "^0.1.0", "@protobufs/tendermint": "^0.0.10", - "@subql/common": "^4.1.0", + "@subql/common": "^4.1.1", "@subql/types-cosmos": "workspace:*", "@subql/x-cosmology-telescope": "^1.4.14", "fs-extra": "^11.1.1", diff --git a/packages/common-cosmos/src/project/utils.ts b/packages/common-cosmos/src/project/utils.ts index 718786e0f..ce80e1ff0 100644 --- a/packages/common-cosmos/src/project/utils.ts +++ b/packages/common-cosmos/src/project/utils.ts @@ -1,7 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {FileReference} from '@subql/types-core'; +import {FileReference, BaseTemplateDataSource} from '@subql/types-core'; import { SecondLayerHandlerProcessor, CosmosCustomDatasource, @@ -15,11 +15,15 @@ import { import {ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface} from 'class-validator'; import {gte} from 'semver'; -export function isCustomCosmosDs(ds: CosmosDatasource): ds is CosmosCustomDatasource { +export function isCustomCosmosDs( + ds: CosmosDatasource | BaseTemplateDataSource +): ds is CosmosCustomDatasource { return ds.kind !== CosmosDatasourceKind.Runtime && !!(ds as CosmosCustomDatasource).processor; } -export function isRuntimeCosmosDs(ds: CosmosDatasource): ds is CosmosRuntimeDatasource { +export function isRuntimeCosmosDs( + ds: CosmosDatasource | BaseTemplateDataSource +): ds is CosmosRuntimeDatasource { return ds.kind === CosmosDatasourceKind.Runtime; } diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index a3e943e28..ddf22fa89 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed - Update cosmos and subql dependencies (#270) -- Update to latest `@subql/node-core` (#272) +- Use Subquery Project code from node core +- Breaking change: Update to latest `@subql/node-core`, require indexing environment timezone set to UTC (#272) + +### Fixed +- Fix testing service missing inject nodeConfig ### Added - Detection of Cosmos SDK version to use correct client type (#270) diff --git a/packages/node/Dockerfile b/packages/node/Dockerfile index 225f36817..5bb4c143c 100644 --- a/packages/node/Dockerfile +++ b/packages/node/Dockerfile @@ -1,6 +1,6 @@ # Build stage FROM node:18-alpine as builder - +ENV TZ ='UTC' # Set working directory WORKDIR /app diff --git a/packages/node/package.json b/packages/node/package.json index aab593bae..f14fc3f8e 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -29,9 +29,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^4.1.0", + "@subql/common": "^4.1.1", "@subql/common-cosmos": "workspace:*", - "@subql/node-core": "^12.0.0", + "@subql/node-core": "^13.0.0", "@subql/types-cosmos": "workspace:*", "lodash": "^4.17.21", "protobufjs": "^6.11.4", diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 926ae3949..ee26c25b8 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -1,29 +1,18 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import assert from 'assert'; import fs from 'fs'; import os from 'os'; import { sep } from 'path'; import { isMainThread } from 'worker_threads'; -import { Injectable } from '@nestjs/common'; -import { validateSemver } from '@subql/common'; import { CosmosProjectNetworkConfig, parseCosmosProjectManifest, - ProjectManifestV1_0_0Impl, - isRuntimeCosmosDs, - isCustomCosmosDs, + isRuntimeCosmosDs as isRuntimeDs, + isCustomCosmosDs as isCustomDs, } from '@subql/common-cosmos'; -import { - CronFilter, - insertBlockFiltersCronSchedules, - ISubqueryProject, - loadProjectTemplates, - updateDataSourcesV1_0_0, - WorkerHost, -} from '@subql/node-core'; -import { ParentProject, Reader, RunnerSpecs } from '@subql/types-core'; +import { CronFilter, WorkerHost, BaseSubqueryProject } from '@subql/node-core'; +import { Reader } from '@subql/types-core'; import { CosmosDatasource, CustomDatasourceTemplate, @@ -31,9 +20,6 @@ import { CosmosHandlerKind, CosmosBlockFilter, } from '@subql/types-cosmos'; -import { buildSchemaFromString } from '@subql/utils'; -import { GraphQLSchema } from 'graphql'; -import { processNetworkConfig } from '../utils/project'; const { version: packageVersion } = require('../../package.json'); @@ -43,81 +29,40 @@ export type CosmosProjectDsTemplate = export type SubqlProjectBlockFilter = CosmosBlockFilter & CronFilter; -const NOT_SUPPORT = (name: string) => { - throw new Error(`Manifest specVersion ${name} is not supported`); -}; - // This is the runtime type after we have mapped genesisHash to chainId and endpoint/dict have been provided when dealing with deployments type NetworkConfig = CosmosProjectNetworkConfig & { chainId: string }; -@Injectable() -export class SubqueryProject implements ISubqueryProject { - #dataSources: CosmosDatasource[]; - - constructor( - readonly id: string, - readonly root: string, - readonly network: NetworkConfig, - dataSources: CosmosDatasource[], - readonly schema: GraphQLSchema, - readonly templates: CosmosProjectDsTemplate[], - readonly runner?: RunnerSpecs, - readonly parent?: ParentProject, - readonly tempDir?: string, - ) { - this.#dataSources = dataSources; - } - - get dataSources(): CosmosDatasource[] { - return this.#dataSources; - } - - async applyCronTimestamps( - getTimestamp: (height: number) => Promise, - ): Promise { - this.#dataSources = await insertBlockFiltersCronSchedules( - this.dataSources, - getTimestamp, - isRuntimeCosmosDs, - CosmosHandlerKind.Block, - ); - } - - static async create( - path: string, - rawManifest: unknown, - reader: Reader, - root: string, - networkOverrides?: Partial, - ): Promise { - // rawManifest and reader can be reused here. - // It has been pre-fetched and used for rebase manifest runner options with args - // in order to generate correct configs. - - // But we still need reader here, because path can be remote or local - // and the `loadProjectManifest(projectPath)` only support local mode - if (rawManifest === undefined) { - throw new Error(`Get manifest from project path ${path} failed`); - } +export type SubqueryProject = BaseSubqueryProject< + CosmosDatasource, + CosmosProjectDsTemplate, + NetworkConfig +> & { tempDir?: string }; - const manifest = parseCosmosProjectManifest(rawManifest); +export async function createSubQueryProject( + path: string, + rawManifest: unknown, + reader: Reader, + root: string, // If project local then directory otherwise temp directory + networkOverrides?: Partial, +): Promise { + const project = await BaseSubqueryProject.create({ + parseManifest: (raw) => parseCosmosProjectManifest(raw).asV1_0_0, + path, + rawManifest, + reader, + root, + nodeSemver: packageVersion, + blockHandlerKind: CosmosHandlerKind.Block, + networkOverrides, + isRuntimeDs, + isCustomDs, + }); - if (!manifest.isV1_0_0) { - NOT_SUPPORT('<1.0.0'); - } + project.tempDir = getTempDir(); - return loadProjectFromManifestBase( - manifest.asV1_0_0, - reader, - path, - root, - networkOverrides, - ); - } + return project; } -type SUPPORT_MANIFEST = ProjectManifestV1_0_0Impl; - /** * Gets a temp dir shared between main thread and workers * */ @@ -134,72 +79,3 @@ function getTempDir(): string { } return workerTempDir; } - -async function loadProjectFromManifestBase( - projectManifest: SUPPORT_MANIFEST, - reader: Reader, - path: string, - root: string, - networkOverrides?: Partial, -): Promise { - if (typeof projectManifest.network.endpoint === 'string') { - projectManifest.network.endpoint = [projectManifest.network.endpoint]; - } - - const network = await processNetworkConfig( - { - ...projectManifest.network, - ...networkOverrides, - }, - reader, - ); - - if (!network.endpoint) { - throw new Error( - `Network endpoint must be provided for network. chainId="${network.chainId}"`, - ); - } - - let schemaString: string; - try { - schemaString = await reader.getFile(projectManifest.schema.file); - } catch (e) { - throw new Error( - `unable to fetch the schema from ${projectManifest.schema.file}`, - ); - } - const schema = buildSchemaFromString(schemaString); - - const dataSources = await updateDataSourcesV1_0_0( - projectManifest.dataSources, - reader, - root, - isCustomCosmosDs, - ); - - const templates = await loadProjectTemplates( - projectManifest.templates, - root, - reader, - isCustomCosmosDs, - ); - const runner = projectManifest.runner; - assert( - validateSemver(packageVersion, runner.node.version), - new Error( - `Runner require node version ${runner.node.version}, current node ${packageVersion}`, - ), - ); - - return new SubqueryProject( - reader.root ? reader.root : path, //TODO, need to method to get project_id - root, - network, - dataSources, - schema, - templates, - runner, - projectManifest.parent, - getTempDir(), - ); -} diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts index de172d9b3..4716d8c00 100644 --- a/packages/node/src/configure/configure.module.ts +++ b/packages/node/src/configure/configure.module.ts @@ -4,7 +4,7 @@ import { DynamicModule, Global, Module } from '@nestjs/common'; import { NodeConfig, registerApp } from '@subql/node-core'; import { yargsOptions } from '../yargs'; -import { SubqueryProject } from './SubqueryProject'; +import { createSubQueryProject, SubqueryProject } from './SubqueryProject'; const pjson = require('../../package.json'); @@ -18,7 +18,7 @@ export class ConfigureModule { const { argv } = yargsOptions; return registerApp( argv, - SubqueryProject.create.bind(SubqueryProject), + createSubQueryProject, yargsOptions.showHelp.bind(yargsOptions), pjson, ); diff --git a/packages/node/src/indexer/dynamic-ds.service.spec.ts b/packages/node/src/indexer/dynamic-ds.service.spec.ts index 63a5840a4..ca82bd1ca 100644 --- a/packages/node/src/indexer/dynamic-ds.service.spec.ts +++ b/packages/node/src/indexer/dynamic-ds.service.spec.ts @@ -27,13 +27,11 @@ describe('Creating dynamic ds', () => { let project: SubqueryProject; beforeEach(async () => { - project = new SubqueryProject( - '', - '', - null, - [], - null, - [ + project = { + id: '', + root: '', + dataSources: [], + templates: [ { name: 'cosmos', kind: CosmosDatasourceKind.Runtime, @@ -77,10 +75,7 @@ describe('Creating dynamic ds', () => { }, }, ], - null, - null, - null, - ); + } as unknown as SubqueryProject; dynamiDsService = new DynamicDsService(null, project); await dynamiDsService.init(getMetadata()); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 0282ad527..aaaea2f3e 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -62,7 +62,12 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; await apiService.init(); return apiService; }, - inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2], + inject: [ + 'ISubqueryProject', + ConnectionPoolService, + EventEmitter2, + NodeConfig, + ], }, SchedulerRegistry, TestRunner, diff --git a/yarn.lock b/yarn.lock index c8569da43..6c29f5544 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6362,7 +6362,7 @@ __metadata: "@protobufs/google": ^0.0.10 "@protobufs/ibc": ^0.1.0 "@protobufs/tendermint": ^0.0.10 - "@subql/common": ^4.1.0 + "@subql/common": ^4.1.1 "@subql/types-cosmos": "workspace:*" "@subql/x-cosmology-telescope": ^1.4.14 "@types/bn.js": 4.11.6 @@ -6385,9 +6385,9 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:4.1.0, @subql/common@npm:^4.1.0": - version: 4.1.0 - resolution: "@subql/common@npm:4.1.0" +"@subql/common@npm:4.1.1, @subql/common@npm:^4.1.1": + version: 4.1.1 + resolution: "@subql/common@npm:4.1.1" dependencies: "@subql/types-core": 0.10.0 axios: ^0.28.0 @@ -6398,22 +6398,22 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 8beac982f53ad99efb9a5725d73768f945a4fb52ff50222c3db047e4143013a9e716c0cb63d561dd8ae40fa2f0a8ceb472516d6e3fc2beb58ee888361a7798a7 + checksum: 2f578678b5af8190639ada7e2f22401b3ca7b7c7cab1bfdeb5860365c14e6c6a8724b02c95f1f8fedfcc01205d02d3de45cd486e2c3ee10ff5f898f973d6e0dc languageName: node linkType: hard -"@subql/node-core@npm:^12.0.0": - version: 12.0.0 - resolution: "@subql/node-core@npm:12.0.0" +"@subql/node-core@npm:^13.0.0": + version: 13.0.0 + resolution: "@subql/node-core@npm:13.0.0" dependencies: "@apollo/client": ^3.8.8 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 - "@subql/common": 4.1.0 + "@subql/common": 4.1.1 "@subql/testing": 2.2.1 "@subql/types": 3.9.0 - "@subql/utils": 2.13.0 + "@subql/utils": 2.13.1 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 async-mutex: ^0.4.0 @@ -6431,7 +6431,7 @@ __metadata: toposort-class: ^1.0.1 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: a37ab972f75dee4ecda070aafb125194a03952bcc8c6f2dacbae4c8c4f97cb5228b53ab09614d24658ad576f8f9de7a5d07c491f288ea1f449a2f80040235731 + checksum: 91ebd7999da59470ef13923db694b81353e810892d63aa15174ff1c2688e61d6b0860fb53efdb31700fccf0772a42fbe19afff66efac5fb9ca258c9aa4012b2f languageName: node linkType: hard @@ -6451,9 +6451,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^4.1.0 + "@subql/common": ^4.1.1 "@subql/common-cosmos": "workspace:*" - "@subql/node-core": ^12.0.0 + "@subql/node-core": ^13.0.0 "@subql/types-cosmos": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -6524,9 +6524,9 @@ __metadata: languageName: node linkType: hard -"@subql/utils@npm:2.13.0": - version: 2.13.0 - resolution: "@subql/utils@npm:2.13.0" +"@subql/utils@npm:2.13.1": + version: 2.13.1 + resolution: "@subql/utils@npm:2.13.1" dependencies: "@polkadot/util": ^12.6.2 "@polkadot/util-crypto": ^12.6.2 @@ -6542,7 +6542,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.2.1 - checksum: e2c43c0c2865f5e014572da3ec0a41691e5c534f90b0f008f12770ab9871783bba348cb929f5d51e36527d11344107718c3f86ef1b227dbc344ee03233c48031 + checksum: eab0f5d4933dc6c56bbadbd82235c37beeaa9789372187ff523e08b46b41ce39eff406cfd6dfb955e396e0183def56901ef52396fb5f778b1247762a7e95a439 languageName: node linkType: hard