Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

partners[patch]: Fix flaky token counting tests #6000

Merged
merged 4 commits into from
Jul 8, 2024
Merged

partners[patch]: Fix flaky token counting tests #6000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
16be57d
partners[patch]: Fix flaky token counting tests
bracesproul Jul 8, 2024
790cb40
chore: lint files
bracesproul Jul 8, 2024
bb4b19b
Merge branch 'main' into brace/fix-token-tests
bracesproul Jul 8, 2024
6a3483b
Merge branch 'main' into brace/fix-token-tests
bracesproul Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions libs/langchain-anthropic/src/tests/chat_models.int.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,16 @@ import {
import { CallbackManager } from "@langchain/core/callbacks/manager";
import { ChatAnthropic } from "../chat_models.js";

async function sleep(ms = 1000): Promise<void> {
return new Promise<void>((resolve) => {
setTimeout(resolve, ms);
});
}

afterEach(async () => {
await sleep();
});
Comment on lines +16 to +24
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

b/c anthropic rate limits. will remove if we can get our limits increased

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed here #6003


test("Test ChatAnthropic", async () => {
const chat = new ChatAnthropic({
modelName: "claude-3-sonnet-20240229",
Expand Down Expand Up @@ -323,6 +333,7 @@ test("Stream tokens", async () => {
const model = new ChatAnthropic({
model: "claude-3-haiku-20240307",
temperature: 0,
maxTokens: 10,
});
let res: AIMessageChunk | null = null;
for await (const chunk of await model.stream(
Expand All @@ -339,8 +350,8 @@ test("Stream tokens", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(34);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down
4 changes: 2 additions & 2 deletions libs/langchain-cohere/src/tests/chat_models.int.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ test("Stream token count usage_metadata", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(71);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
Expand Down Expand Up @@ -134,7 +134,7 @@ test("Invoke token count usage_metadata", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(71);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
Expand Down
10 changes: 6 additions & 4 deletions libs/langchain-google-genai/src/tests/chat_models.int.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,7 @@ test("ChatGoogleGenerativeAI can call withStructuredOutput genai tools and invok
test("Stream token count usage_metadata", async () => {
const model = new ChatGoogleGenerativeAI({
temperature: 0,
maxOutputTokens: 10,
});
let res: AIMessageChunk | null = null;
for await (const chunk of await model.stream(
Expand All @@ -461,8 +462,8 @@ test("Stream token count usage_metadata", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(10);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down Expand Up @@ -490,15 +491,16 @@ test("streamUsage excludes token usage", async () => {
test("Invoke token count usage_metadata", async () => {
const model = new ChatGoogleGenerativeAI({
temperature: 0,
maxOutputTokens: 10,
});
const res = await model.invoke("Why is the sky blue? Be concise.");
console.log(res);
expect(res?.usage_metadata).toBeDefined();
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(10);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down
10 changes: 6 additions & 4 deletions libs/langchain-google-vertexai/src/tests/chat_models.int.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ describe("GAuth Chat", () => {
test("Stream token count usage_metadata", async () => {
const model = new ChatVertexAI({
temperature: 0,
maxOutputTokens: 10,
});
let res: AIMessageChunk | null = null;
for await (const chunk of await model.stream(
Expand All @@ -253,8 +254,8 @@ test("Stream token count usage_metadata", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(9);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down Expand Up @@ -282,15 +283,16 @@ test("streamUsage excludes token usage", async () => {
test("Invoke token count usage_metadata", async () => {
const model = new ChatVertexAI({
temperature: 0,
maxOutputTokens: 10,
});
const res = await model.invoke("Why is the sky blue? Be concise.");
console.log(res);
expect(res?.usage_metadata).toBeDefined();
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(9);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down
10 changes: 6 additions & 4 deletions libs/langchain-mistralai/src/tests/chat_models.int.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,7 @@ test("Stream token count usage_metadata", async () => {
const model = new ChatMistralAI({
model: "codestral-latest",
temperature: 0,
maxTokens: 10,
});
let res: AIMessageChunk | null = null;
for await (const chunk of await model.stream(
Expand All @@ -938,8 +939,8 @@ test("Stream token count usage_metadata", async () => {
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(13);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down Expand Up @@ -969,15 +970,16 @@ test("Invoke token count usage_metadata", async () => {
const model = new ChatMistralAI({
model: "codestral-latest",
temperature: 0,
maxTokens: 10,
});
const res = await model.invoke("Why is the sky blue? Be concise.");
console.log(res);
expect(res?.usage_metadata).toBeDefined();
if (!res?.usage_metadata) {
return;
}
expect(res.usage_metadata.input_tokens).toBe(13);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(10);
expect(res.usage_metadata.input_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.output_tokens).toBeGreaterThan(1);
expect(res.usage_metadata.total_tokens).toBe(
res.usage_metadata.input_tokens + res.usage_metadata.output_tokens
);
Expand Down
Loading