feat: vectordb modules (#2)

cli/config.yml

@@ -6,11 +6,15 @@ logLevel: info # error| info | warn | error | debug | silly
 llmProvider: ollama # openai | ollama
 
 tools:
-  - name: codeSummaryQuery
+  codeSummaryQuery:
     module: '@interrobangc/codellm-tool-code-summary-query'
-  - name: docSummaryQuery
+    config:
+      vectorDbName: chromadb
+  docSummaryQuery:
     module: '@interrobangc/codellm-tool-doc-summary-query'
-  - name: jsDependencyTree
+    config:
+      vectorDbName: chromadb
+  jsDependencyTree:
     module: '@interrobangc/codellm-tool-js-dependency-tree'
 
 # The configuration for individual providers can be set here. You probably want to use

cli/package.json

@@ -17,6 +17,7 @@
     "@interrobangc/codellm": "^0.0.1",
     "@interrobangc/codellm-tool-code-summary-query": "^0.0.1",
     "@interrobangc/codellm-tool-js-dependency-tree": "^0.0.1",
+    "@interrobangc/codellm-vectordb-chromadb": "^0.0.1",
     "@langchain/openai": "^0.0.15",
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",

codellm/package.json

@@ -14,11 +14,10 @@
     "test:unit": "jest --coverage '.*\\.test\\.ts'"
   },
   "dependencies": {
-    "chromadb": "^1.8.1",
+    "@langchain/core": "^0.1.40",
     "chromadb-default-embed": "^2.13.2",
     "crypto": "^1.0.1",
     "globby": "^14.0.1",
-    "langchain": "^0.1.24",
     "lodash": "^4.17.21",
     "winston": "^3.11.0"
   },

codellm/src/config/constants.ts

@@ -92,14 +92,20 @@ export const DEFAULTS = {
       },
     },
   },
-  tools: [
-    {
-      name: 'codeSummaryQuery',
+  tools: {
+    codeSummaryQuery: {
       module: '@interrobangc/codellm-tool-code-summary-query',
+      config: {
+        vectorDbName: 'chromadb',
+      },
     },
-  ],
+  },
 
-  vectorDb: 'chromadb',
+  vectorDbs: {
+    chromadb: {
+      module: '@interrobangc/codellm-vectordb-chromadb',
+    },
+  },
 } as const;
 
 /* eslint-enable sonarjs/no-duplicate-string */

codellm/src/config/types.ts

@@ -6,8 +6,8 @@ import type {
   Provider,
   ProviderConfigs,
   ProviderServiceItem,
-  ToolConfigItem,
-  VectorDb,
+  ToolConfigs,
+  VectorDbConfigs,
 } from '@/.';
 
 export type Service = (typeof SERVICES_TYPE)[keyof typeof SERVICES_TYPE];
@@ -17,16 +17,17 @@ export type ConfigCommon = {
   logFormat: LogFormat;
   logLevel: LogLevel;
   path: string;
-  vectorDb: VectorDb;
-  tools?: ToolConfigItem[];
+  tools?: ToolConfigs;
 };
 
 export type Config = ConfigCommon & {
   llms: Record<Service, ProviderServiceItem>;
   providers: ProviderConfigs;
+  vectorDbs: VectorDbConfigs;
 };
 
 export type PartialConfig = Partial<ConfigCommon> & {
   llms?: Partial<Record<Service, ProviderServiceItem>>;
   providers?: ProviderConfigs;
+  vectorDbs?: VectorDbConfigs;
 };

codellm/src/tool/index.ts

@@ -18,9 +18,9 @@ export const initTools = async (config: Config): Promise<Tools> => {
 
   const tools: Tools = {};
 
-  const toolInits = config.tools!.map(async (tool) => {
+  const toolInits = Object.entries(config.tools).map(async ([name, tool]) => {
     const toolModule = await import(tool.module);
-    tools[tool.name] = await toolModule.newTool(tool.name, config);
+    tools[name] = await toolModule.newTool(name, config);
   });
 
   await Promise.all(toolInits);

codellm/src/tool/types.ts

@@ -22,11 +22,12 @@ export type ToolRunReturn = {
 export type ToolConfig = Record<string, unknown>;
 
 export type ToolConfigItem = {
-  name: string;
   module: string;
   config: ToolConfig;
 };
 
+export type ToolConfigs = Record<string, ToolConfigItem>;
+
 export type ToolDescriptionParamsType = 'bool' | 'string' | 'number';
 
 export type ToolDescriptionParams = {
@@ -49,3 +50,8 @@ export type Tool = {
   description: ToolDescription;
 };
 export type Tools = Record<string, Tool>;
+
+export type VectorDbToolConfig = {
+  vectorDbName: string;
+  vectorDbCollectionName: string;
+};

codellm/src/vectorDb/index.ts

@@ -1,34 +1,49 @@
-import type { Config, VectorDb, VectorDbClient } from '@/.';
-import { VECTOR_DB_MODULES } from './constants.js';
+import type { Config, VectorDb, VectorDbClient, VectorDbs } from '@/.';
+import log from '@/log/index.js';
+
+const vectorDbs: VectorDbs = {};
 
 /**
  * Import a VectorDb module from the vectorDb library
  *
- * @param dbName - The name of the VectorDb to import.
+ * @param {VectorDb} name - The name of the VectorDb to import.
+ * @param {VectorDbConfigItem} config - The configuration for the VectorDb.
  *
  * @returns The VectorDb module.
- *
- * @throws If the VectorDb is not found.
- * @throws If there is an error importing the VectorDb.
- */
-export const importVectorDb = async (dbName: VectorDb) => {
-  const dbModule = VECTOR_DB_MODULES[dbName];
-  if (!dbModule) throw new Error(`VectorDb not found: ${dbName}`);
+ **/
+export const importVectorDbModule = async (name: VectorDb, config: Config) => {
+  const dbConfig = config.vectorDbs[name];
+
+  if (!dbConfig) throw new Error(`VectorDb config not found: ${name}`);
+  const dbModule = dbConfig.module;
 
   return import(dbModule);
 };
 
 /**
  * Create a new VectorDbClient instance from the vectorDb library
  *
- * @param config - The configuration object.
+ * @param {VectorDb} name - The name of the VectorDb to get.
+ * @param {Config} config - The configuration object.
  *
  * @returns The new VectorDbClient instance.
  */
-export const newClient = async (config: Config): Promise<VectorDbClient> => {
-  const db = await importVectorDb(config.vectorDb);
-  return db.newClient();
+export const newClient = async (
+  name: VectorDb,
+  config: Config,
+): Promise<VectorDbClient> => {
+  log(`vectordb newClient: ${name}`, 'silly');
+
+  if (!vectorDbs[name]) {
+    const module = await importVectorDbModule(name, config);
+    vectorDbs[name] = module.newClient();
+  }
+
+  if (!vectorDbs[name]) {
+    throw new Error(`VectorDb not found: ${name}`);
+  }
+
+  return vectorDbs[name]!;
 };
 
 export * from './types.js';
-export * from './constants.js';

codellm/src/vectorDb/types.ts

@@ -1,8 +1,15 @@
 import type { Document, Embedding, Metadata } from 'chromadb';
 
-import type { VECTOR_DBS } from './constants';
+export type VectorDb = string;
 
-export type VectorDb = (typeof VECTOR_DBS)[keyof typeof VECTOR_DBS];
+export type VectorDbClientConfig = Record<string, unknown>;
+
+export type VectorDbConfigItem = {
+  module: string;
+  config: VectorDbClientConfig;
+};
+
+export type VectorDbConfigs = Record<VectorDb, VectorDbConfigItem>;
 
 export type EmbeddingDocument = {
   id: string;
@@ -51,3 +58,5 @@ export type VectorDbClient = {
   query: (params: VectorDbQueryParams) => Promise<unknown>;
   get: (params: VectorDbGetParams) => Promise<unknown>;
 };
+
+export type VectorDbs = Record<VectorDb, VectorDbClient>;