feat: add assertion methods (#1)

lucgagan · Nov 14, 2023 · 01fc156 · 01fc156
1 parent e126dd3
commit 01fc156
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 2 deletions.
diff --git a/src/completeTask.ts b/src/completeTask.ts
@@ -5,6 +5,10 @@ import { createActions } from "./createActions";
 
 const defaultDebug = process.env.AUTO_PLAYWRIGHT_DEBUG === "true";
 
+type Snapshot = {
+  steps: {name: string, arguments: string}[];
+};
+
 export const completeTask = async (
   page: Page,
   task: TaskMessage
@@ -18,6 +22,10 @@ export const completeTask = async (
 
   const debug = task.options?.debug?? defaultDebug;
 
+  const snapshot: Snapshot = {
+    steps: [],
+  };
+
   const runner = openai.beta.chat.completions
     .runFunctions({
       model: task.options?.model ?? "gpt-4-1106-preview",
@@ -29,6 +37,10 @@ export const completeTask = async (
         console.log("> message", message);
       }
 
+      if (message.role === 'assistant' && message.function_call) {
+        snapshot.steps.push(message.function_call);
+      }
+
       if (
         message.role === "assistant" &&
         message.function_call?.name.startsWith("result")
@@ -51,5 +63,7 @@ export const completeTask = async (
     console.log("> lastFunctionResult", lastFunctionResult);
   }
 
+  console.log(snapshot);
+
   return lastFunctionResult;
 };
diff --git a/src/createActions.ts b/src/createActions.ts
@@ -492,6 +492,66 @@ export const createActions = (page: Page): (RunnableFunctionWithoutParse | Runna
         },
       },
     },
+    {
+      function: (args: {actual: string; expected: string}) => {
+        return {
+          actual: args.actual,
+          expected: args.expected,
+          success: args.actual === args.expected,
+        };
+      },
+      name: "expect_toBe",
+      description: "Asserts that the actual value is equal to the expected value.",
+      parse: (args: string) => {
+        return z
+          .object({
+            actual: z.string(),
+            expected: z.string(),
+          })
+          .parse(JSON.parse(args));
+      },
+      parameters: {
+        type: "object",
+        properties: {
+          actual: {
+            type: "string",
+          },
+          expected: {
+            type: "string",
+          },
+        },
+      },
+    },
+    {
+      function: (args: {actual: string; expected: string}) => {
+        return {
+          actual: args.actual,
+          expected: args.expected,
+          success: args.actual !== args.expected,
+        };
+      },
+      name: "expect_notToBe",
+      description: "Asserts that the actual value is not equal to the expected value.",
+      parse: (args: string) => {
+        return z
+          .object({
+            actual: z.string(),
+            expected: z.string(),
+          })
+          .parse(JSON.parse(args));
+      },
+      parameters: {
+        type: "object",
+        properties: {
+          actual: {
+            type: "string",
+          },
+          expected: {
+            type: "string",
+          },
+        },
+      },
+    },
     {
       function: (args: { assertion: boolean }) => {
         return args;

diff --git a/tests/auto.spec.ts b/tests/auto.spec.ts
@@ -30,7 +30,7 @@ test("executes click", async ({ page }) => {
   await expect(page.getByTestId("current-count")).toHaveText("2");
 });
 
-test("asserts (positive)", async ({ page }) => {
+test("asserts (toBe)", async ({ page }) => {
   await page.goto("/");
 
   const searchInputHasHeaderText = await auto(
@@ -41,7 +41,7 @@ test("asserts (positive)", async ({ page }) => {
   expect(searchInputHasHeaderText).toBe(true);
 });
 
-test("asserts (negative)", async ({ page }) => {
+test("asserts (not.toBe)", async ({ page }) => {
   await page.goto("/");
 
   const searchInputHasHeaderText = await auto(