Fix test fw missing config (#273)

* Fix testing service missing inject nodeConfig * Sync with main, fix testing service * Fix build issue --------- Co-authored-by: Scott Twiname <skott.twiname@gmail.com>
subquery · Jul 26, 2024 · 6e44e01 · 6e44e01
1 parent 6174b4c
commit 6e44e01
Show file tree

Hide file tree

Showing 10 changed files with 77 additions and 193 deletions.
diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json
@@ -22,7 +22,7 @@
     "@protobufs/google": "^0.0.10",
     "@protobufs/ibc": "^0.1.0",
     "@protobufs/tendermint": "^0.0.10",
-    "@subql/common": "^4.1.0",
+    "@subql/common": "^4.1.1",
     "@subql/types-cosmos": "workspace:*",
     "@subql/x-cosmology-telescope": "^1.4.14",
     "fs-extra": "^11.1.1",

diff --git a/packages/common-cosmos/src/project/utils.ts b/packages/common-cosmos/src/project/utils.ts
@@ -1,7 +1,7 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {FileReference} from '@subql/types-core';
+import {FileReference, BaseTemplateDataSource} from '@subql/types-core';
 import {
   SecondLayerHandlerProcessor,
   CosmosCustomDatasource,
@@ -15,11 +15,15 @@ import {
 import {ValidationArguments, ValidatorConstraint, ValidatorConstraintInterface} from 'class-validator';
 import {gte} from 'semver';
 
-export function isCustomCosmosDs(ds: CosmosDatasource): ds is CosmosCustomDatasource<string> {
+export function isCustomCosmosDs(
+  ds: CosmosDatasource | BaseTemplateDataSource<CosmosDatasource>
+): ds is CosmosCustomDatasource<string> {
   return ds.kind !== CosmosDatasourceKind.Runtime && !!(ds as CosmosCustomDatasource<string>).processor;
 }
 
-export function isRuntimeCosmosDs(ds: CosmosDatasource): ds is CosmosRuntimeDatasource {
+export function isRuntimeCosmosDs(
+  ds: CosmosDatasource | BaseTemplateDataSource<CosmosDatasource>
+): ds is CosmosRuntimeDatasource {
   return ds.kind === CosmosDatasourceKind.Runtime;
 }
 

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Changed
 - Update cosmos and subql dependencies (#270)
-- Update to latest `@subql/node-core` (#272)
+- Use Subquery Project code from node core 
+- Breaking change: Update to latest `@subql/node-core`, require indexing environment timezone set to UTC (#272)
+
+### Fixed
+- Fix testing service missing inject nodeConfig 
 
 ### Added
 - Detection of Cosmos SDK version to use correct client type (#270)

diff --git a/packages/node/Dockerfile b/packages/node/Dockerfile
@@ -1,6 +1,6 @@
 # Build stage
 FROM node:18-alpine as builder
-
+ENV TZ ='UTC'
 # Set working directory
 WORKDIR /app
 

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -29,9 +29,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^4.1.0",
+    "@subql/common": "^4.1.1",
     "@subql/common-cosmos": "workspace:*",
-    "@subql/node-core": "^12.0.0",
+    "@subql/node-core": "^13.0.0",
     "@subql/types-cosmos": "workspace:*",
     "lodash": "^4.17.21",
     "protobufjs": "^6.11.4",

diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts
@@ -1,39 +1,25 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import assert from 'assert';
 import fs from 'fs';
 import os from 'os';
 import { sep } from 'path';
 import { isMainThread } from 'worker_threads';
-import { Injectable } from '@nestjs/common';
-import { validateSemver } from '@subql/common';
 import {
   CosmosProjectNetworkConfig,
   parseCosmosProjectManifest,
-  ProjectManifestV1_0_0Impl,
-  isRuntimeCosmosDs,
-  isCustomCosmosDs,
+  isRuntimeCosmosDs as isRuntimeDs,
+  isCustomCosmosDs as isCustomDs,
 } from '@subql/common-cosmos';
-import {
-  CronFilter,
-  insertBlockFiltersCronSchedules,
-  ISubqueryProject,
-  loadProjectTemplates,
-  updateDataSourcesV1_0_0,
-  WorkerHost,
-} from '@subql/node-core';
-import { ParentProject, Reader, RunnerSpecs } from '@subql/types-core';
+import { CronFilter, WorkerHost, BaseSubqueryProject } from '@subql/node-core';
+import { Reader } from '@subql/types-core';
 import {
   CosmosDatasource,
   CustomDatasourceTemplate,
   RuntimeDatasourceTemplate,
   CosmosHandlerKind,
   CosmosBlockFilter,
 } from '@subql/types-cosmos';
-import { buildSchemaFromString } from '@subql/utils';
-import { GraphQLSchema } from 'graphql';
-import { processNetworkConfig } from '../utils/project';
 
 const { version: packageVersion } = require('../../package.json');
 
@@ -43,81 +29,40 @@ export type CosmosProjectDsTemplate =
 
 export type SubqlProjectBlockFilter = CosmosBlockFilter & CronFilter;
 
-const NOT_SUPPORT = (name: string) => {
-  throw new Error(`Manifest specVersion ${name} is not supported`);
-};
-
 // This is the runtime type after we have mapped genesisHash to chainId and endpoint/dict have been provided when dealing with deployments
 type NetworkConfig = CosmosProjectNetworkConfig & { chainId: string };
 
-@Injectable()
-export class SubqueryProject implements ISubqueryProject {
-  #dataSources: CosmosDatasource[];
-
-  constructor(
-    readonly id: string,
-    readonly root: string,
-    readonly network: NetworkConfig,
-    dataSources: CosmosDatasource[],
-    readonly schema: GraphQLSchema,
-    readonly templates: CosmosProjectDsTemplate[],
-    readonly runner?: RunnerSpecs,
-    readonly parent?: ParentProject,
-    readonly tempDir?: string,
-  ) {
-    this.#dataSources = dataSources;
-  }
-
-  get dataSources(): CosmosDatasource[] {
-    return this.#dataSources;
-  }
-
-  async applyCronTimestamps(
-    getTimestamp: (height: number) => Promise<Date>,
-  ): Promise<void> {
-    this.#dataSources = await insertBlockFiltersCronSchedules(
-      this.dataSources,
-      getTimestamp,
-      isRuntimeCosmosDs,
-      CosmosHandlerKind.Block,
-    );
-  }
-
-  static async create(
-    path: string,
-    rawManifest: unknown,
-    reader: Reader,
-    root: string,
-    networkOverrides?: Partial<CosmosProjectNetworkConfig>,
-  ): Promise<SubqueryProject> {
-    // rawManifest and reader can be reused here.
-    // It has been pre-fetched and used for rebase manifest runner options with args
-    // in order to generate correct configs.
-
-    // But we still need reader here, because path can be remote or local
-    // and the `loadProjectManifest(projectPath)` only support local mode
-    if (rawManifest === undefined) {
-      throw new Error(`Get manifest from project path ${path} failed`);
-    }
+export type SubqueryProject = BaseSubqueryProject<
+  CosmosDatasource,
+  CosmosProjectDsTemplate,
+  NetworkConfig
+> & { tempDir?: string };
 
-    const manifest = parseCosmosProjectManifest(rawManifest);
+export async function createSubQueryProject(
+  path: string,
+  rawManifest: unknown,
+  reader: Reader,
+  root: string, // If project local then directory otherwise temp directory
+  networkOverrides?: Partial<NetworkConfig>,
+): Promise<SubqueryProject> {
+  const project = await BaseSubqueryProject.create<SubqueryProject>({
+    parseManifest: (raw) => parseCosmosProjectManifest(raw).asV1_0_0,
+    path,
+    rawManifest,
+    reader,
+    root,
+    nodeSemver: packageVersion,
+    blockHandlerKind: CosmosHandlerKind.Block,
+    networkOverrides,
+    isRuntimeDs,
+    isCustomDs,
+  });
 
-    if (!manifest.isV1_0_0) {
-      NOT_SUPPORT('<1.0.0');
-    }
+  project.tempDir = getTempDir();
 
-    return loadProjectFromManifestBase(
-      manifest.asV1_0_0,
-      reader,
-      path,
-      root,
-      networkOverrides,
-    );
-  }
+  return project;
 }
 
-type SUPPORT_MANIFEST = ProjectManifestV1_0_0Impl;
-
 /**
  * Gets a temp dir shared between main thread and workers
  * */
@@ -134,72 +79,3 @@ function getTempDir(): string {
   }
   return workerTempDir;
 }
-
-async function loadProjectFromManifestBase(
-  projectManifest: SUPPORT_MANIFEST,
-  reader: Reader,
-  path: string,
-  root: string,
-  networkOverrides?: Partial<CosmosProjectNetworkConfig>,
-): Promise<SubqueryProject> {
-  if (typeof projectManifest.network.endpoint === 'string') {
-    projectManifest.network.endpoint = [projectManifest.network.endpoint];
-  }
-
-  const network = await processNetworkConfig(
-    {
-      ...projectManifest.network,
-      ...networkOverrides,
-    },
-    reader,
-  );
-
-  if (!network.endpoint) {
-    throw new Error(
-      `Network endpoint must be provided for network. chainId="${network.chainId}"`,
-    );
-  }
-
-  let schemaString: string;
-  try {
-    schemaString = await reader.getFile(projectManifest.schema.file);
-  } catch (e) {
-    throw new Error(
-      `unable to fetch the schema from ${projectManifest.schema.file}`,
-    );
-  }
-  const schema = buildSchemaFromString(schemaString);
-
-  const dataSources = await updateDataSourcesV1_0_0(
-    projectManifest.dataSources,
-    reader,
-    root,
-    isCustomCosmosDs,
-  );
-
-  const templates = await loadProjectTemplates(
-    projectManifest.templates,
-    root,
-    reader,
-    isCustomCosmosDs,
-  );
-  const runner = projectManifest.runner;
-  assert(
-    validateSemver(packageVersion, runner.node.version),
-    new Error(
-      `Runner require node version ${runner.node.version}, current node ${packageVersion}`,
-    ),
-  );
-
-  return new SubqueryProject(
-    reader.root ? reader.root : path, //TODO, need to method to get project_id
-    root,
-    network,
-    dataSources,
-    schema,
-    templates,
-    runner,
-    projectManifest.parent,
-    getTempDir(),
-  );
-}
diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts
@@ -4,7 +4,7 @@
 import { DynamicModule, Global, Module } from '@nestjs/common';
 import { NodeConfig, registerApp } from '@subql/node-core';
 import { yargsOptions } from '../yargs';
-import { SubqueryProject } from './SubqueryProject';
+import { createSubQueryProject, SubqueryProject } from './SubqueryProject';
 
 const pjson = require('../../package.json');
 
@@ -18,7 +18,7 @@ export class ConfigureModule {
     const { argv } = yargsOptions;
     return registerApp<SubqueryProject>(
       argv,
-      SubqueryProject.create.bind(SubqueryProject),
+      createSubQueryProject,
       yargsOptions.showHelp.bind(yargsOptions),
       pjson,
     );

diff --git a/packages/node/src/indexer/dynamic-ds.service.spec.ts b/packages/node/src/indexer/dynamic-ds.service.spec.ts
@@ -27,13 +27,11 @@ describe('Creating dynamic ds', () => {
   let project: SubqueryProject;
 
   beforeEach(async () => {
-    project = new SubqueryProject(
-      '',
-      '',
-      null,
-      [],
-      null,
-      [
+    project = {
+      id: '',
+      root: '',
+      dataSources: [],
+      templates: [
         {
           name: 'cosmos',
           kind: CosmosDatasourceKind.Runtime,
@@ -77,10 +75,7 @@ describe('Creating dynamic ds', () => {
           },
         },
       ],
-      null,
-      null,
-      null,
-    );
+    } as unknown as SubqueryProject;
     dynamiDsService = new DynamicDsService(null, project);
 
     await dynamiDsService.init(getMetadata());

diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts
@@ -62,7 +62,12 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
         await apiService.init();
         return apiService;
       },
-      inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2],
+      inject: [
+        'ISubqueryProject',
+        ConnectionPoolService,
+        EventEmitter2,
+        NodeConfig,
+      ],
     },
     SchedulerRegistry,
     TestRunner,