🐛 fix: fix claude 3.5 image with s3 url (#3870)

* ✅ test: add tests for imageToBase64 * 🐛 fix: fix anthropic image chat * ✅ test: add more tests
lobehub · Sep 9, 2024 · 89c8dd4 · 89c8dd4
1 parent 831bc5f
commit 89c8dd4
Show file tree

Hide file tree

Showing 5 changed files with 240 additions and 59 deletions.
diff --git a/src/libs/agent-runtime/anthropic/index.ts b/src/libs/agent-runtime/anthropic/index.ts
@@ -29,7 +29,8 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
-      const anthropicPayload = this.buildAnthropicPayload(payload);
+      const anthropicPayload = await this.buildAnthropicPayload(payload);
+
       const response = await this.client.messages.create(
         { ...anthropicPayload, stream: true },
         {
@@ -86,20 +87,17 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
     }
   }
 
-  private buildAnthropicPayload(payload: ChatStreamPayload) {
+  private async buildAnthropicPayload(payload: ChatStreamPayload) {
     const { messages, model, max_tokens = 4096, temperature, top_p, tools } = payload;
     const system_message = messages.find((m) => m.role === 'system');
     const user_messages = messages.filter((m) => m.role !== 'system');
 
     return {
       max_tokens,
-      messages: buildAnthropicMessages(user_messages),
+      messages: await buildAnthropicMessages(user_messages),
       model,
       system: system_message?.content as string,
-      temperature: 
-        payload.temperature !== undefined 
-        ? temperature / 2
-        : undefined,
+      temperature: payload.temperature !== undefined ? temperature / 2 : undefined,
       tools: buildAnthropicTools(tools),
       top_p,
     } satisfies Anthropic.MessageCreateParams;

diff --git a/src/libs/agent-runtime/bedrock/index.ts b/src/libs/agent-runtime/bedrock/index.ts
@@ -62,7 +62,7 @@ export class LobeBedrockAI implements LobeRuntimeAI {
       body: JSON.stringify({
         anthropic_version: 'bedrock-2023-05-31',
         max_tokens: max_tokens || 4096,
-        messages: buildAnthropicMessages(user_messages),
+        messages: await buildAnthropicMessages(user_messages),
         system: system_message?.content as string,
         temperature: temperature / 2,
         tools: buildAnthropicTools(tools),

diff --git a/src/libs/agent-runtime/utils/anthropicHelpers.test.ts b/src/libs/agent-runtime/utils/anthropicHelpers.test.ts
@@ -1,6 +1,8 @@
 import { OpenAI } from 'openai';
 import { describe, expect, it } from 'vitest';
 
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
+
 import { OpenAIChatMessage, UserMessageContentPart } from '../types/chat';
 import {
   buildAnthropicBlock,
@@ -10,28 +12,30 @@ import {
 } from './anthropicHelpers';
 import { parseDataUri } from './uriParser';
 
-describe('anthropicHelpers', () => {
-  // Mock the parseDataUri function since it's an implementation detail
-  vi.mock('./uriParser', () => ({
-    parseDataUri: vi.fn().mockReturnValue({
-      mimeType: 'image/jpeg',
-      base64: 'base64EncodedString',
-    }),
-  }));
+// Mock the parseDataUri function since it's an implementation detail
+vi.mock('./uriParser', () => ({
+  parseDataUri: vi.fn().mockReturnValue({
+    mimeType: 'image/jpeg',
+    base64: 'base64EncodedString',
+    type: 'base64',
+  }),
+}));
+vi.mock('@/utils/imageToBase64');
 
+describe('anthropicHelpers', () => {
   describe('buildAnthropicBlock', () => {
-    it('should return the content as is for text type', () => {
+    it('should return the content as is for text type', async () => {
       const content: UserMessageContentPart = { type: 'text', text: 'Hello!' };
-      const result = buildAnthropicBlock(content);
+      const result = await buildAnthropicBlock(content);
       expect(result).toEqual(content);
     });
 
-    it('should transform an image URL into an Anthropic.ImageBlockParam', () => {
+    it('should transform an image URL into an Anthropic.ImageBlockParam', async () => {
       const content: UserMessageContentPart = {
         type: 'image_url',
         image_url: { url: 'data:image/jpeg;base64,base64EncodedString' },
       };
-      const result = buildAnthropicBlock(content);
+      const result = await buildAnthropicBlock(content);
       expect(parseDataUri).toHaveBeenCalledWith(content.image_url.url);
       expect(result).toEqual({
         source: {
@@ -42,48 +46,116 @@ describe('anthropicHelpers', () => {
         type: 'image',
       });
     });
+
+    it('should transform a regular image URL into an Anthropic.ImageBlockParam', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: 'image/png',
+        base64: null,
+        type: 'url',
+      });
+      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.png' },
+      } as const;
+
+      const result = await buildAnthropicBlock(content);
+
+      expect(parseDataUri).toHaveBeenCalledWith(content.image_url.url);
+      expect(imageUrlToBase64).toHaveBeenCalledWith(content.image_url.url);
+      expect(result).toEqual({
+        source: {
+          data: 'convertedBase64String',
+          media_type: 'image/png',
+          type: 'base64',
+        },
+        type: 'image',
+      });
+    });
+
+    it('should use default media_type for URL images when mimeType is not provided', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: null,
+        base64: null,
+        type: 'url',
+      });
+      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image' },
+      } as const;
+
+      const result = await buildAnthropicBlock(content);
+
+      expect(result).toEqual({
+        source: {
+          data: 'convertedBase64String',
+          media_type: 'image/png',
+          type: 'base64',
+        },
+        type: 'image',
+      });
+    });
+
+    it('should throw an error for invalid image URLs', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: null,
+        base64: null,
+        // @ts-ignore
+        type: 'invalid',
+      });
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'invalid-url' },
+      } as const;
+
+      await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
+    });
   });
 
   describe('buildAnthropicMessage', () => {
-    it('should correctly convert system message to assistant message', () => {
+    it('should correctly convert system message to assistant message', async () => {
       const message: OpenAIChatMessage = {
         content: [{ type: 'text', text: 'Hello!' }],
         role: 'system',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({ content: [{ type: 'text', text: 'Hello!' }], role: 'user' });
     });
 
-    it('should correctly convert user message with string content', () => {
+    it('should correctly convert user message with string content', async () => {
       const message: OpenAIChatMessage = {
         content: 'Hello!',
         role: 'user',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({ content: 'Hello!', role: 'user' });
     });
 
-    it('should correctly convert user message with content parts', () => {
+    it('should correctly convert user message with content parts', async () => {
       const message: OpenAIChatMessage = {
         content: [
           { type: 'text', text: 'Check out this image:' },
           { type: 'image_url', image_url: { url: 'data:image/png;base64,abc123' } },
         ],
         role: 'user',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('user');
       expect(result.content).toHaveLength(2);
       expect((result.content[1] as any).type).toBe('image');
     });
 
-    it('should correctly convert tool message', () => {
+    it('should correctly convert tool message', async () => {
       const message: OpenAIChatMessage = {
         content: 'Tool result content',
         role: 'tool',
         tool_call_id: 'tool123',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('user');
       expect(result.content).toEqual([
         {
@@ -94,7 +166,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert assistant message with tool calls', () => {
+    it('should correctly convert assistant message with tool calls', async () => {
       const message: OpenAIChatMessage = {
         content: 'Here is the result:',
         role: 'assistant',
@@ -109,7 +181,7 @@ describe('anthropicHelpers', () => {
           },
         ],
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('assistant');
       expect(result.content).toEqual([
         { text: 'Here is the result:', type: 'text' },
@@ -122,12 +194,12 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert function message', () => {
+    it('should correctly convert function message', async () => {
       const message: OpenAIChatMessage = {
         content: 'def hello(name):\n  return f"Hello {name}"',
         role: 'function',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({
         content: 'def hello(name):\n  return f"Hello {name}"',
         role: 'assistant',
@@ -136,28 +208,28 @@ describe('anthropicHelpers', () => {
   });
 
   describe('buildAnthropicMessages', () => {
-    it('should correctly convert OpenAI Messages to Anthropic Messages', () => {
+    it('should correctly convert OpenAI Messages to Anthropic Messages', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'Hello', role: 'user' },
         { content: 'Hi', role: 'assistant' },
       ];
 
-      const result = buildAnthropicMessages(messages);
+      const result = await buildAnthropicMessages(messages);
       expect(result).toHaveLength(2);
       expect(result).toEqual([
         { content: 'Hello', role: 'user' },
         { content: 'Hi', role: 'assistant' },
       ]);
     });
 
-    it('messages should end with user', () => {
+    it('messages should end with user', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'Hello', role: 'user' },
         { content: 'Hello', role: 'user' },
         { content: 'Hi', role: 'assistant' },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toHaveLength(4);
       expect(contents).toEqual([
@@ -168,7 +240,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('messages should pair', () => {
+    it('messages should pair', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'a', role: 'assistant' },
         { content: 'b', role: 'assistant' },
@@ -177,7 +249,7 @@ describe('anthropicHelpers', () => {
         { content: '你好', role: 'user' },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toHaveLength(9);
       expect(contents).toEqual([
@@ -193,7 +265,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert OpenAI tool message to Anthropic format', () => {
+    it('should correctly convert OpenAI tool message to Anthropic format', async () => {
       const messages: OpenAIChatMessage[] = [
         {
           content: '告诉我杭州和北京的天气，先回答我好的',
@@ -242,7 +314,7 @@ describe('anthropicHelpers', () => {
         },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toEqual([
         { content: '告诉我杭州和北京的天气，先回答我好的', role: 'user' },