Merge branch 'main' of https://github.com/lobehub/lobe-chat

CHANGELOG.md

@@ -2,6 +2,56 @@
 
 # Changelog
 
+### [Version 1.15.30](https://github.com/lobehub/lobe-chat/compare/v1.15.29...v1.15.30)
+
+<sup>Released on **2024-09-09**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Fix claude 3.5 image with s3 url.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Fix claude 3.5 image with s3 url, closes [#3870](https://github.com/lobehub/lobe-chat/issues/3870) ([89c8dd4](https://github.com/lobehub/lobe-chat/commit/89c8dd4))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.15.29](https://github.com/lobehub/lobe-chat/compare/v1.15.28...v1.15.29)
+
+<sup>Released on **2024-09-09**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Gemini cannot input images when server database is enabled.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Gemini cannot input images when server database is enabled, closes [#3370](https://github.com/lobehub/lobe-chat/issues/3370) ([eb552d2](https://github.com/lobehub/lobe-chat/commit/eb552d2))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.15.28](https://github.com/lobehub/lobe-chat/compare/v1.15.27...v1.15.28)
 
 <sup>Released on **2024-09-09**</sup>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.15.28",
+  "version": "1.15.30",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

src/libs/agent-runtime/anthropic/index.ts

@@ -29,7 +29,8 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
-      const anthropicPayload = this.buildAnthropicPayload(payload);
+      const anthropicPayload = await this.buildAnthropicPayload(payload);
+
       const response = await this.client.messages.create(
         { ...anthropicPayload, stream: true },
         {
@@ -86,20 +87,17 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
     }
   }
 
-  private buildAnthropicPayload(payload: ChatStreamPayload) {
+  private async buildAnthropicPayload(payload: ChatStreamPayload) {
     const { messages, model, max_tokens = 4096, temperature, top_p, tools } = payload;
     const system_message = messages.find((m) => m.role === 'system');
     const user_messages = messages.filter((m) => m.role !== 'system');
 
     return {
       max_tokens,
-      messages: buildAnthropicMessages(user_messages),
+      messages: await buildAnthropicMessages(user_messages),
       model,
       system: system_message?.content as string,
-      temperature: 
-        payload.temperature !== undefined 
-        ? temperature / 2
-        : undefined,
+      temperature: payload.temperature !== undefined ? temperature / 2 : undefined,
       tools: buildAnthropicTools(tools),
       top_p,
     } satisfies Anthropic.MessageCreateParams;

src/libs/agent-runtime/bedrock/index.ts

@@ -62,7 +62,7 @@ export class LobeBedrockAI implements LobeRuntimeAI {
       body: JSON.stringify({
         anthropic_version: 'bedrock-2023-05-31',
         max_tokens: max_tokens || 4096,
-        messages: buildAnthropicMessages(user_messages),
+        messages: await buildAnthropicMessages(user_messages),
         system: system_message?.content as string,
         temperature: temperature / 2,
         tools: buildAnthropicTools(tools),

src/libs/agent-runtime/google/index.test.ts

@@ -5,6 +5,7 @@ import OpenAI from 'openai';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import { OpenAIChatMessage } from '@/libs/agent-runtime';
+import * as imageToBase64Module from '@/utils/imageToBase64';
 
 import * as debugStreamModule from '../utils/debugStream';
 import { LobeGoogleAI } from './index';
@@ -303,36 +304,57 @@ describe('LobeGoogleAI', () => {
 
   describe('private method', () => {
     describe('convertContentToGooglePart', () => {
-      it('should throw TypeError when image URL does not contain base64 data', () => {
-        // 提供一个不包含base64数据的图像URL
-        const invalidImageUrl = 'http://example.com/image.png';
+      it('should handle URL type images', async () => {
+        const imageUrl = 'http://example.com/image.png';
+        const mockBase64 = 'mockBase64Data';
 
-        expect(() =>
+        // Mock the imageUrlToBase64 function
+        vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce(mockBase64);
+
+        const result = await instance['convertContentToGooglePart']({
+          type: 'image_url',
+          image_url: { url: imageUrl },
+        });
+
+        expect(result).toEqual({
+          inlineData: {
+            data: mockBase64,
+            mimeType: 'image/png',
+          },
+        });
+
+        expect(imageToBase64Module.imageUrlToBase64).toHaveBeenCalledWith(imageUrl);
+      });
+
+      it('should throw TypeError for unsupported image URL types', async () => {
+        const unsupportedImageUrl = 'unsupported://example.com/image.png';
+
+        await expect(
           instance['convertContentToGooglePart']({
             type: 'image_url',
-            image_url: { url: invalidImageUrl },
+            image_url: { url: unsupportedImageUrl },
           }),
-        ).toThrow(TypeError);
+        ).rejects.toThrow(TypeError);
       });
     });
 
     describe('buildGoogleMessages', () => {
-      it('get default result with gemini-pro', () => {
+      it('get default result with gemini-pro', async () => {
         const messages: OpenAIChatMessage[] = [{ content: 'Hello', role: 'user' }];
 
-        const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
+        const contents = await instance['buildGoogleMessages'](messages, 'gemini-pro');
 
         expect(contents).toHaveLength(1);
         expect(contents).toEqual([{ parts: [{ text: 'Hello' }], role: 'user' }]);
       });
 
-      it('messages should end with user if using gemini-pro', () => {
+      it('messages should end with user if using gemini-pro', async () => {
         const messages: OpenAIChatMessage[] = [
           { content: 'Hello', role: 'user' },
           { content: 'Hi', role: 'assistant' },
         ];
 
-        const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
+        const contents = await instance['buildGoogleMessages'](messages, 'gemini-pro');
 
         expect(contents).toHaveLength(3);
         expect(contents).toEqual([
@@ -342,13 +364,13 @@ describe('LobeGoogleAI', () => {
         ]);
       });
 
-      it('should include system role if there is a system role prompt', () => {
+      it('should include system role if there is a system role prompt', async () => {
         const messages: OpenAIChatMessage[] = [
           { content: 'you are ChatGPT', role: 'system' },
           { content: 'Who are you', role: 'user' },
         ];
 
-        const contents = instance['buildGoogleMessages'](messages, 'gemini-pro');
+        const contents = await instance['buildGoogleMessages'](messages, 'gemini-pro');
 
         expect(contents).toHaveLength(3);
         expect(contents).toEqual([
@@ -358,13 +380,13 @@ describe('LobeGoogleAI', () => {
         ]);
       });
 
-      it('should not modify the length if model is gemini-1.5-pro', () => {
+      it('should not modify the length if model is gemini-1.5-pro', async () => {
         const messages: OpenAIChatMessage[] = [
           { content: 'Hello', role: 'user' },
           { content: 'Hi', role: 'assistant' },
         ];
 
-        const contents = instance['buildGoogleMessages'](messages, 'gemini-1.5-pro-latest');
+        const contents = await instance['buildGoogleMessages'](messages, 'gemini-1.5-pro-latest');
 
         expect(contents).toHaveLength(2);
         expect(contents).toEqual([
@@ -373,7 +395,7 @@ describe('LobeGoogleAI', () => {
         ]);
       });
 
-      it('should use specified model when images are included in messages', () => {
+      it('should use specified model when images are included in messages', async () => {
         const messages: OpenAIChatMessage[] = [
           {
             content: [
@@ -386,7 +408,7 @@ describe('LobeGoogleAI', () => {
         const model = 'gemini-1.5-flash-latest';
 
         // 调用 buildGoogleMessages 方法
-        const contents = instance['buildGoogleMessages'](messages, model);
+        const contents = await instance['buildGoogleMessages'](messages, model);
 
         expect(contents).toHaveLength(1);
         expect(contents).toEqual([
@@ -501,35 +523,35 @@ describe('LobeGoogleAI', () => {
     });
 
     describe('convertOAIMessagesToGoogleMessage', () => {
-      it('should correctly convert assistant message', () => {
+      it('should correctly convert assistant message', async () => {
         const message: OpenAIChatMessage = {
           role: 'assistant',
           content: 'Hello',
         };
 
-        const converted = instance['convertOAIMessagesToGoogleMessage'](message);
+        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
 
         expect(converted).toEqual({
           role: 'model',
           parts: [{ text: 'Hello' }],
         });
       });
 
-      it('should correctly convert user message', () => {
+      it('should correctly convert user message', async () => {
         const message: OpenAIChatMessage = {
           role: 'user',
           content: 'Hi',
         };
 
-        const converted = instance['convertOAIMessagesToGoogleMessage'](message);
+        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
 
         expect(converted).toEqual({
           role: 'user',
           parts: [{ text: 'Hi' }],
         });
       });
 
-      it('should correctly convert message with inline base64 image parts', () => {
+      it('should correctly convert message with inline base64 image parts', async () => {
         const message: OpenAIChatMessage = {
           role: 'user',
           content: [
@@ -538,7 +560,7 @@ describe('LobeGoogleAI', () => {
           ],
         };
 
-        const converted = instance['convertOAIMessagesToGoogleMessage'](message);
+        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
 
         expect(converted).toEqual({
           role: 'user',
@@ -548,7 +570,7 @@ describe('LobeGoogleAI', () => {
           ],
         });
       });
-      it.skip('should correctly convert message with image url parts', () => {
+      it.skip('should correctly convert message with image url parts', async () => {
         const message: OpenAIChatMessage = {
           role: 'user',
           content: [
@@ -557,7 +579,7 @@ describe('LobeGoogleAI', () => {
           ],
         };
 
-        const converted = instance['convertOAIMessagesToGoogleMessage'](message);
+        const converted = await instance['convertOAIMessagesToGoogleMessage'](message);
 
         expect(converted).toEqual({
           role: 'user',