diff --git a/cortex-js/package.json b/cortex-js/package.json index 76207f3e8..52214d2e0 100644 --- a/cortex-js/package.json +++ b/cortex-js/package.json @@ -38,7 +38,7 @@ "preuninstall": "node ./uninstall.js" }, "dependencies": { - "@cortexso/cortex.js": "^0.1.4", + "@cortexso/cortex.js": "^0.1.5", "@huggingface/gguf": "^0.1.5", "@huggingface/hub": "^0.15.1", "@nestjs/axios": "^3.0.2", diff --git a/cortex-js/src/infrastructure/commanders/run.command.ts b/cortex-js/src/infrastructure/commanders/run.command.ts index 5aefcf732..45416847f 100644 --- a/cortex-js/src/infrastructure/commanders/run.command.ts +++ b/cortex-js/src/infrastructure/commanders/run.command.ts @@ -57,9 +57,10 @@ export class RunCommand extends BaseCommand { // Check model compatibility on this machine await checkModelCompatibility(modelId, checkingSpinner); + let existingModel = await this.cortex.models.retrieve(modelId); // If not exist // Try Pull - if (!(await this.cortex.models.retrieve(modelId))) { + if (!existingModel) { checkingSpinner.succeed(); console.log('Downloading model...'); @@ -68,16 +69,17 @@ export class RunCommand extends BaseCommand { exit(1); }); await downloadProgress(this.cortex, modelId); - } - ora().succeed('Model downloaded'); + checkingSpinner.succeed('Model downloaded'); - // Second check if model is available - const existingModel = await this.cortex.models.retrieve(modelId); - if (!existingModel) { - checkingSpinner.fail(`Model is not available`); - process.exit(1); + // Second check if model is available + existingModel = await this.cortex.models.retrieve(modelId); + if (!existingModel) { + checkingSpinner.fail(`Model is not available`); + process.exit(1); + } + } else { + checkingSpinner.succeed('Model found'); } - checkingSpinner.succeed('Model found'); const engine = existingModel.engine || Engines.llamaCPP; // Pull engine if not exist diff --git a/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts b/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts index b320ff2b1..ca3932ecd 100644 --- a/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts +++ b/cortex-js/src/infrastructure/commanders/test/models.command.spec.ts @@ -1,5 +1,4 @@ import { TestingModule } from '@nestjs/testing'; -import { stubMethod } from 'hanbi'; import { CommandTestFactory } from 'nest-commander-testing'; import { CommandModule } from '@/command.module'; import { join } from 'path'; diff --git a/cortex-js/src/infrastructure/controllers/embeddings.controller.spec.ts b/cortex-js/src/infrastructure/controllers/embeddings.controller.spec.ts index c5b47d604..f60087678 100644 --- a/cortex-js/src/infrastructure/controllers/embeddings.controller.spec.ts +++ b/cortex-js/src/infrastructure/controllers/embeddings.controller.spec.ts @@ -10,6 +10,7 @@ import { EventEmitterModule } from '@nestjs/event-emitter'; import { TelemetryModule } from '@/usecases/telemetry/telemetry.module'; import { FileManagerModule } from '../services/file-manager/file-manager.module'; import { ModelsModule } from '@/usecases/models/models.module'; +import { CortexModule } from '@/usecases/cortex/cortex.module'; describe('EmbeddingsController', () => { let controller: EmbeddingsController; @@ -27,6 +28,7 @@ describe('EmbeddingsController', () => { TelemetryModule, FileManagerModule, ModelsModule, + CortexModule, ], controllers: [EmbeddingsController], providers: [ChatUsecases], diff --git a/cortex-js/src/infrastructure/controllers/models.controller.ts b/cortex-js/src/infrastructure/controllers/models.controller.ts index 0aa6a87c5..71933d005 100644 --- a/cortex-js/src/infrastructure/controllers/models.controller.ts +++ b/cortex-js/src/infrastructure/controllers/models.controller.ts @@ -19,7 +19,6 @@ import { DeleteModelResponseDto } from '@/infrastructure/dtos/models/delete-mode import { ApiOperation, ApiParam, ApiTags, ApiResponse } from '@nestjs/swagger'; import { StartModelSuccessDto } from '@/infrastructure/dtos/models/start-model-success.dto'; import { TransformInterceptor } from '../interceptors/transform.interceptor'; -import { CortexUsecases } from '@/usecases/cortex/cortex.usecases'; import { ModelSettingsDto } from '../dtos/models/model-settings.dto'; import { EventName } from '@/domain/telemetry/telemetry.interface'; import { TelemetryUsecases } from '@/usecases/telemetry/telemetry.usecases'; @@ -32,7 +31,6 @@ import { HuggingFaceRepoSibling } from '@/domain/models/huggingface.interface'; export class ModelsController { constructor( private readonly modelsUsecases: ModelsUsecases, - private readonly cortexUsecases: CortexUsecases, private readonly telemetryUsecases: TelemetryUsecases, ) {} @@ -71,9 +69,7 @@ export class ModelsController { @Param('modelId') modelId: string, @Body() params: ModelSettingsDto, ) { - return this.cortexUsecases - .startCortex() - .then(() => this.modelsUsecases.startModel(modelId, params)); + return this.modelsUsecases.startModel(modelId, params); } @HttpCode(200) diff --git a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts index c71d344d5..8f9a9ce67 100644 --- a/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts +++ b/cortex-js/src/infrastructure/repositories/extensions/extension.repository.ts @@ -28,19 +28,46 @@ export class ExtensionRepositoryImpl implements ExtensionRepository { this.loadCoreExtensions(); this.loadExternalExtensions(); } + /** + * Persist extension to the extensions map + * @param object + * @returns + */ create(object: Extension): Promise { this.extensions.set(object.name ?? '', object); return Promise.resolve(object); } + + /** + * Find all extensions + * @returns + */ findAll(): Promise { return Promise.resolve(Array.from(this.extensions.values())); } + + /** + * Find one extension by id + * @param id + * @returns + */ findOne(id: string): Promise { return Promise.resolve(this.extensions.get(id) ?? null); } + + /** + * Update extension + * It is not applicable + */ update(): Promise { throw new Error('Method not implemented.'); } + + /** + * Remove extension from the extensions map + * @param id + * @returns + */ remove(id: string): Promise { this.extensions.delete(id); return Promise.resolve(); diff --git a/cortex-js/src/usecases/cortex/cortex.usecases.ts b/cortex-js/src/usecases/cortex/cortex.usecases.ts index 08162de34..da05e2918 100644 --- a/cortex-js/src/usecases/cortex/cortex.usecases.ts +++ b/cortex-js/src/usecases/cortex/cortex.usecases.ts @@ -73,6 +73,12 @@ export class CortexUsecases implements BeforeApplicationShutdown { }); this.cortexProcess.unref(); + // Handle process exit + this.cortexProcess.on('close', (code) => { + this.cortexProcess = undefined + console.log(`child process exited with code ${code}`); + }); + // Await for the /healthz status ok return new Promise((resolve, reject) => { const interval = setInterval(() => { diff --git a/cortex-js/src/usecases/models/models.usecases.spec.ts b/cortex-js/src/usecases/models/models.usecases.spec.ts index 8bea2eaa3..ab6a13d09 100644 --- a/cortex-js/src/usecases/models/models.usecases.spec.ts +++ b/cortex-js/src/usecases/models/models.usecases.spec.ts @@ -10,6 +10,7 @@ import { DownloadManagerModule } from '@/infrastructure/services/download-manage import { EventEmitterModule } from '@nestjs/event-emitter'; import { TelemetryModule } from '../telemetry/telemetry.module'; import { ContextModule } from '@/infrastructure/services/context/context.module'; +import { CortexModule } from '../cortex/cortex.module'; describe('ModelsService', () => { let service: ModelsUsecases; @@ -30,6 +31,7 @@ describe('ModelsService', () => { TelemetryModule, TelemetryModule, ContextModule, + CortexModule, ], providers: [ModelsUsecases], exports: [ModelsUsecases], diff --git a/cortex-js/src/usecases/models/models.usecases.ts b/cortex-js/src/usecases/models/models.usecases.ts index cf1fc8185..1f67f35f6 100644 --- a/cortex-js/src/usecases/models/models.usecases.ts +++ b/cortex-js/src/usecases/models/models.usecases.ts @@ -34,11 +34,13 @@ import { ContextService } from '@/infrastructure/services/context/context.servic import { Engines } from '@/infrastructure/commanders/types/engine.interface'; import { load } from 'js-yaml'; import { llamaModelFile } from '@/utils/app-path'; +import { CortexUsecases } from '../cortex/cortex.usecases'; @Injectable() export class ModelsUsecases { constructor( private readonly modelRepository: ModelRepository, + private readonly cortexUsecases: CortexUsecases, private readonly extensionRepository: ExtensionRepository, private readonly fileManagerService: FileManagerService, private readonly downloadManagerService: DownloadManagerService, @@ -170,6 +172,10 @@ export class ModelsUsecases { modelId, }; } + + // Attempt to start cortex + await this.cortexUsecases.startCortex() + const loadingModelSpinner = ora('Loading model...').start(); // update states and emitting event this.activeModelStatuses[modelId] = {