ChatAnthropic streamEvents not working with bindTools #5906
Replies: 2 comments 1 reply
-
The reason no "on_chat_model_stream" events are emitted when using the In the Here is a relevant excerpt from the async *_streamResponseChunks(
messages: BaseMessage[],
options: this["ParsedCallOptions"],
runManager?: CallbackManagerForLLMRun
): AsyncGenerator<ChatGenerationChunk> {
const params = this.invocationParams(options);
const formattedMessages = _formatMessagesForAnthropic(messages);
if (options.tools !== undefined && options.tools.length > 0) {
const { generations } = await this._generateNonStreaming(
messages,
params,
{
signal: options.signal,
}
);
const result = generations[0].message as AIMessage;
const toolCallChunks = result.tool_calls?.map(
(toolCall: ToolCall, index: number) => ({
name: toolCall.name,
args: JSON.stringify(toolCall.args),
id: toolCall.id,
index,
})
);
yield new ChatGenerationChunk({
message: new AIMessageChunk({
content: result.content,
additional_kwargs: result.additional_kwargs,
tool_call_chunks: toolCallChunks,
}),
text: generations[0].text,
});
} else {
const stream = await this.createStreamWithRetry({
...params,
...formattedMessages,
stream: true,
});
let usageData = { input_tokens: 0, output_tokens: 0 };
for await (const data of stream) {
if (options.signal?.aborted) {
stream.controller.abort();
throw new Error("AbortError: User aborted the request.");
}
if (data.type === "message_start") {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { content, usage, ...additionalKwargs } = data.message;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const filteredAdditionalKwargs: Record<string, any> = {};
for (const [key, value] of Object.entries(additionalKwargs)) {
if (value !== undefined && value !== null) {
filteredAdditionalKwargs[key] = value;
} The To emit "on_chat_model_stream" events, you would need to modify the implementation to include event emission logic within the streaming process. This could involve adding calls to an event emitter or callback manager at appropriate points in the For example, in the async *_streamIterator(
input: BaseLanguageModelInput,
options?: CallOptions
): AsyncGenerator<OutputMessageType> {
// Subclass check required to avoid double callbacks with default implementation
if (
this._streamResponseChunks ===
BaseChatModel.prototype._streamResponseChunks
) {
yield this.invoke(input, options);
} else {
const prompt = BaseChatModel._convertInputToPromptValue(input);
const messages = prompt.toChatMessages();
const [runnableConfig, callOptions] =
this._separateRunnableConfigFromCallOptions(options);
const inheritableMetadata = {
...runnableConfig.metadata,
...this.getLsParams(callOptions),
};
const callbackManager_ = await CallbackManager.configure(
runnableConfig.callbacks,
this.callbacks,
runnableConfig.tags,
this.tags,
inheritableMetadata,
this.metadata,
{ verbose: this.verbose }
);
const extra = {
options: callOptions,
invocation_params: this?.invocationParams(callOptions),
batch_size: 1,
};
const runManagers = await callbackManager_?.handleChatModelStart(
this.toJSON(),
[messages],
runnableConfig.runId,
undefined,
extra,
undefined,
undefined,
runnableConfig.runName
);
let generationChunk: ChatGenerationChunk | undefined;
try {
for await (const chunk of this._streamResponseChunks(
messages,
callOptions,
runManagers?.[0]
)) {
chunk.message.response_metadata = {
...chunk.generationInfo,
...chunk.message.response_metadata,
};
yield chunk.message as OutputMessageType;
if (!generationChunk) {
generationChunk = chunk;
} else {
generationChunk = generationChunk.concat(chunk);
}
}
} catch (err) {
await Promise.all(
(runManagers ?? []).map((runManager) =>
runManager?.handleLLMError(err)
)
);
throw err;
}
await Promise.all(
(runManagers ?? []).map((runManager) =>
runManager?.handleLLMEnd({
// TODO: Remove cast after figuring out inheritance
generations: [[generationChunk as ChatGeneration]],
})
)
);
}
} In this example, the |
Beta Was this translation helpful? Give feedback.
-
What it is recommended way to stream the final output when using ChatAnthropic with tools? |
Beta Was this translation helpful? Give feedback.
-
Checked other resources
Commit to Help
Example Code
Description
There is no output because there are no "on_chat_model_stream" events emitted. However if the same code is executed, except the
.bindTools(tools)
is removed from the model's definition, the output is streamed as excepted.System Info
"@langchain/anthropic": "^0.2.2",
"@langchain/community": "^0.2.14",
"@langchain/core": "^0.2.8",
"node": v20.11.1
"npm": 10.2.4
"platform": "linux"
Beta Was this translation helpful? Give feedback.
All reactions