Add guides for msg format and llm agents (#8750)

* Add guides

* add changeset

* Add code

* Add code

* Add notebook

* rename msg_format to type

* Fix docs

* notebooks

* missing link

* Update guides

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/true-ears-knock.md

@@ -0,0 +1,5 @@
+---
+"gradio": patch
+---
+
+feat:Add guides for msg format and llm agents

.changeset/young-crabs-begin.md

@@ -13,7 +13,7 @@ highlight:
 
 Building Gradio applications around these LLM solutions is now even easier! 
 
-`gr.Chatbot` and `gr.ChatInterface` now have a `msg_format` parameter that can accept two values - `'tuples'` and `'messages'`. If set to `'tuples'`, the default chatbot data format is expected. If set to `'messages'`, a list of dictionaries with `content` and `role` keys is expected. See below - 
+`gr.Chatbot` and `gr.ChatInterface` now have a `type` parameter that can accept two values - `'tuples'` and `'messages'`. If set to `'tuples'`, the default chatbot data format is expected. If set to `'messages'`, a list of dictionaries with `content` and `role` keys is expected. See below - 
 
 ```python
 def chat_greeter(msg, history):
@@ -80,7 +80,7 @@ def generate_response(history):
 
 
 with gr.Blocks() as demo:
-    chatbot  = gr.Chatbot(msg_format="messages")
+    chatbot  = gr.Chatbot(type="messages")
     button = gr.Button("Get San Francisco Weather")
     button.click(generate_response, chatbot, chatbot)
 

client/python/test/conftest.py

@@ -464,7 +464,7 @@ def max_file_size_demo():
 @pytest.fixture
 def chatbot_message_format():
     with gr.Blocks() as demo:
-        chatbot = gr.Chatbot(msg_format="messages")
+        chatbot = gr.Chatbot(type="messages")
         msg = gr.Textbox()
 
         def respond(message, chat_history: list):

demo/agent_chatbot/requirements.txt

@@ -0,0 +1 @@
+git+https://github.com/huggingface/transformers.git#egg=transformers[agents]

demo/agent_chatbot/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: agent_chatbot"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio git+https://github.com/huggingface/transformers.git#egg=transformers[agents]"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "!wget -q https://github.com/gradio-app/gradio/raw/main/demo/agent_chatbot/utils.py"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from gradio import ChatMessage\n", "from transformers import load_tool, ReactCodeAgent, HfEngine\n", "from utils import stream_from_transformers_agent\n", "\n", "# Import tool from Hub\n", "image_generation_tool = load_tool(\"m-ric/text-to-image\")\n", "\n", "\n", "llm_engine = HfEngine(\"meta-llama/Meta-Llama-3-70B-Instruct\")\n", "# Initialize the agent with both tools\n", "agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine)\n", "\n", "\n", "def interact_with_agent(prompt, messages):\n", "    messages.append(ChatMessage(role=\"user\", content=prompt))\n", "    yield messages\n", "    for msg in stream_from_transformers_agent(agent, prompt):\n", "        messages.append(msg)\n", "        yield messages\n", "    yield messages\n", "\n", "\n", "with gr.Blocks() as demo:\n", "    stored_message = gr.State([])\n", "    chatbot = gr.Chatbot(label=\"Agent\",\n", "                         type=\"messages\",\n", "                         avatar_images=(None, \"https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png\"))\n", "    text_input = gr.Textbox(lines=1, label=\"Chat Message\")\n", "    text_input.submit(lambda s: (s, \"\"), [text_input], [stored_message, text_input]).then(interact_with_agent, [stored_message, chatbot], [chatbot])\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/agent_chatbot/run.py

@@ -0,0 +1,34 @@
+import gradio as gr
+from gradio import ChatMessage
+from transformers import load_tool, ReactCodeAgent, HfEngine
+from utils import stream_from_transformers_agent
+
+# Import tool from Hub
+image_generation_tool = load_tool("m-ric/text-to-image")
+
+
+llm_engine = HfEngine("meta-llama/Meta-Llama-3-70B-Instruct")
+# Initialize the agent with both tools
+agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine)
+
+
+def interact_with_agent(prompt, messages):
+    messages.append(ChatMessage(role="user", content=prompt))
+    yield messages
+    for msg in stream_from_transformers_agent(agent, prompt):
+        messages.append(msg)
+        yield messages
+    yield messages
+
+
+with gr.Blocks() as demo:
+    stored_message = gr.State([])
+    chatbot = gr.Chatbot(label="Agent",
+                         type="messages",
+                         avatar_images=(None, "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png"))
+    text_input = gr.Textbox(lines=1, label="Chat Message")
+    text_input.submit(lambda s: (s, ""), [text_input], [stored_message, text_input]).then(interact_with_agent, [stored_message, chatbot], [chatbot])
+
+
+if __name__ == "__main__":
+    demo.launch()

demo/agent_chatbot/utils.py

@@ -0,0 +1,63 @@
+from gradio import ChatMessage
+from transformers.agents import ReactCodeAgent, agent_types
+from typing import Generator
+
+
+def pull_message(step_log: dict):
+    if step_log.get("rationale"):
+        yield ChatMessage(
+            role="assistant", content=step_log["rationale"]
+        )
+    if step_log.get("tool_call"):
+        used_code = step_log["tool_call"]["tool_name"] == "code interpreter"
+        content = step_log["tool_call"]["tool_arguments"]
+        if used_code:
+            content = f"```py\n{content}\n```"
+        yield ChatMessage(
+            role="assistant",
+            metadata={"title": f"🛠️ Used tool {step_log['tool_call']['tool_name']}"},
+            content=content,
+        )
+    if step_log.get("observation"):
+        yield ChatMessage(
+            role="assistant", content=f"```\n{step_log['observation']}\n```"
+        )
+    if step_log.get("error"):
+        yield ChatMessage(
+            role="assistant",
+            content=str(step_log["error"]),
+            metadata={"title": "💥 Error"},
+        )
+
+
+def stream_from_transformers_agent(
+    agent: ReactCodeAgent, prompt: str
+) -> Generator[ChatMessage, None, ChatMessage | None]:
+    """Runs an agent with the given prompt and streams the messages from the agent as ChatMessages."""
+
+    class Output:
+        output: agent_types.AgentType | str = None
+
+    for step_log in agent.run(prompt, stream=True):
+        if isinstance(step_log, dict):
+            for message in pull_message(step_log):
+                print("message", message)
+                yield message
+
+
+    Output.output = step_log
+    if isinstance(Output.output, agent_types.AgentText):
+        yield ChatMessage(
+            role="assistant", content=f"**Final answer:**\n```\n{Output.output.to_string()}\n```")
+    elif isinstance(Output.output, agent_types.AgentImage):
+        yield ChatMessage(
+            role="assistant",
+            content={"path": Output.output.to_string(), "mime_type": "image/png"},
+        )
+    elif isinstance(Output.output, agent_types.AgentAudio):
+        yield ChatMessage(
+            role="assistant",
+            content={"path": Output.output.to_string(), "mime_type": "audio/wav"},
+        )
+    else:
+        return ChatMessage(role="assistant", content=Output.output)

demo/blocks_xray/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: blocks_xray"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import time\n", "\n", "disease_values = [0.25, 0.5, 0.75]\n", "\n", "def xray_model(diseases, img):\n", "    return [{disease: disease_values[idx] for idx,disease in enumerate(diseases)}]\n", "\n", "\n", "def ct_model(diseases, img):\n", "    return [{disease: 0.1 for disease in diseases}]\n", "\n", "with gr.Blocks(fill_width=True) as demo:\n", "    gr.Markdown(\n", "        \"\"\"\n", "# Detect Disease From Scan\n", "With this model you can lorem ipsum\n", "- ipsum 1\n", "- ipsum 2\n", "\"\"\"\n", "    )\n", "    gr.DuplicateButton()\n", "    disease = gr.CheckboxGroup(\n", "        info=\"Select the diseases you want to scan for.\",\n", "        choices=[\"Covid\", \"Malaria\", \"Lung Cancer\"], label=\"Disease to Scan For\"\n", "    )\n", "    slider = gr.Slider(0, 100)\n", "\n", "    with gr.Tab(\"X-ray\") as x_tab:\n", "        with gr.Row():\n", "            xray_scan = gr.Image()\n", "            xray_results = gr.JSON()\n", "        xray_run = gr.Button(\"Run\")\n", "        xray_run.click(\n", "            xray_model,\n", "            inputs=[disease, xray_scan],\n", "            outputs=xray_results,\n", "            api_name=\"xray_model\"\n", "        )\n", "\n", "    with gr.Tab(\"CT Scan\"):\n", "        with gr.Row():\n", "            ct_scan = gr.Image()\n", "            ct_results = gr.JSON()\n", "        ct_run = gr.Button(\"Run\")\n", "        ct_run.click(\n", "            ct_model,\n", "            inputs=[disease, ct_scan],\n", "            outputs=ct_results,\n", "            api_name=\"ct_model\"\n", "        )\n", "\n", "    upload_btn = gr.Button(\"Upload Results\", variant=\"primary\")\n", "    upload_btn.click(\n", "        lambda ct, xr: None,\n", "        inputs=[ct_results, xray_results],\n", "        outputs=[],\n", "    )\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: blocks_xray"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import time\n", "\n", "disease_values = [0.25, 0.5, 0.75]\n", "\n", "def xray_model(diseases, img):\n", "    return [{disease: disease_values[idx] for idx,disease in enumerate(diseases)}]\n", "\n", "\n", "def ct_model(diseases, img):\n", "    return [{disease: 0.1 for disease in diseases}]\n", "\n", "with gr.Blocks() as demo:\n", "    gr.Markdown(\n", "        \"\"\"\n", "# Detect Disease From Scan\n", "With this model you can lorem ipsum\n", "- ipsum 1\n", "- ipsum 2\n", "\"\"\"\n", "    )\n", "    gr.DuplicateButton()\n", "    disease = gr.CheckboxGroup(\n", "        info=\"Select the diseases you want to scan for.\",\n", "        choices=[\"Covid\", \"Malaria\", \"Lung Cancer\"], label=\"Disease to Scan For\"\n", "    )\n", "    slider = gr.Slider(0, 100)\n", "\n", "    with gr.Tab(\"X-ray\") as x_tab:\n", "        with gr.Row():\n", "            xray_scan = gr.Image()\n", "            xray_results = gr.JSON()\n", "        xray_run = gr.Button(\"Run\")\n", "        xray_run.click(\n", "            xray_model,\n", "            inputs=[disease, xray_scan],\n", "            outputs=xray_results,\n", "            api_name=\"xray_model\"\n", "        )\n", "\n", "    with gr.Tab(\"CT Scan\"):\n", "        with gr.Row():\n", "            ct_scan = gr.Image()\n", "            ct_results = gr.JSON()\n", "        ct_run = gr.Button(\"Run\")\n", "        ct_run.click(\n", "            ct_model,\n", "            inputs=[disease, ct_scan],\n", "            outputs=ct_results,\n", "            api_name=\"ct_model\"\n", "        )\n", "\n", "    upload_btn = gr.Button(\"Upload Results\", variant=\"primary\")\n", "    upload_btn.click(\n", "        lambda ct, xr: None,\n", "        inputs=[ct_results, xray_results],\n", "        outputs=[],\n", "    )\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/blocks_xray/run.py

@@ -10,7 +10,7 @@ def xray_model(diseases, img):
 def ct_model(diseases, img):
     return [{disease: 0.1 for disease in diseases}]
 
-with gr.Blocks(fill_width=True) as demo:
+with gr.Blocks() as demo:
     gr.Markdown(
         """
 # Detect Disease From Scan

demo/chatbot_core_components_simple/messages_testcase.py

@@ -65,7 +65,7 @@ def bot(history, response_type):
         elem_id="chatbot",
         bubble_full_width=False,
         scale=1,
-        msg_format="messages"
+        type="messages"
     )
     response_type = gr.Radio(
         [

demo/chatbot_multimodal/messages_testcase.py

@@ -27,7 +27,7 @@ def bot(history: list):
         [],
         elem_id="chatbot",
         bubble_full_width=False,
-        msg_format="messages"
+        type="messages"
     )
 
     chat_input = gr.MultimodalTextbox(interactive=True,

demo/chatbot_streaming/testcase_messages.py

@@ -3,7 +3,7 @@
 import time
 
 with gr.Blocks() as demo:
-    chatbot = gr.Chatbot(msg_format="messages")
+    chatbot = gr.Chatbot(type="messages")
     msg = gr.Textbox()
     clear = gr.Button("Clear")
 

demo/chatbot_with_tools/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatbot_with_tools"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from gradio import ChatMessage\n", "import time\n", "\n", "def generate_response(history):\n", "    history.append(ChatMessage(role=\"user\", content=\"What is the weather in San Francisco right now?\"))\n", "    yield history\n", "    time.sleep(0.25)\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"In order to find the current weather in San Francisco, I will need to use my weather tool.\")\n", "                               )\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"API Error when connecting to weather service.\",\n", "                              metadata={\"title\": \"\ud83d\udca5 Error using tool 'Weather'\"})\n", "                  )\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"I will try again\",\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"Weather 72 degrees Fahrenheit with 20% chance of rain.\",\n", "                                metadata={\"title\": \"\ud83d\udee0\ufe0f Used tool 'Weather'\"}\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"Now that the API succeeded I can complete my task.\",\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"It's a sunny day in San Francisco with a current temperature of 72 degrees Fahrenheit and a 20% chance of rain. Enjoy the weather!\",\n", "                              ))\n", "    yield history\n", "\n", "\n", "with gr.Blocks() as demo:\n", "    chatbot  = gr.Chatbot(msg_format=\"messages\")\n", "    button = gr.Button(\"Get San Francisco Weather\")\n", "    button.click(generate_response, chatbot, chatbot)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatbot_with_tools"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from gradio import ChatMessage\n", "import time\n", "\n", "def generate_response(history):\n", "    history.append(ChatMessage(role=\"user\", content=\"What is the weather in San Francisco right now?\"))\n", "    yield history\n", "    time.sleep(0.25)\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"In order to find the current weather in San Francisco, I will need to use my weather tool.\")\n", "                               )\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"API Error when connecting to weather service.\",\n", "                              metadata={\"title\": \"\ud83d\udca5 Error using tool 'Weather'\"})\n", "                  )\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"I will try again\",\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"Weather 72 degrees Fahrenheit with 20% chance of rain.\",\n", "                                metadata={\"title\": \"\ud83d\udee0\ufe0f Used tool 'Weather'\"}\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"Now that the API succeeded I can complete my task.\",\n", "                              ))\n", "    yield history\n", "    time.sleep(0.25)\n", "\n", "    history.append(ChatMessage(role=\"assistant\",\n", "                               content=\"It's a sunny day in San Francisco with a current temperature of 72 degrees Fahrenheit and a 20% chance of rain. Enjoy the weather!\",\n", "                              ))\n", "    yield history\n", "\n", "\n", "with gr.Blocks() as demo:\n", "    chatbot  = gr.Chatbot(type=\"messages\")\n", "    button = gr.Button(\"Get San Francisco Weather\")\n", "    button.click(generate_response, chatbot, chatbot)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/chatbot_with_tools/run.py

@@ -45,7 +45,7 @@ def generate_response(history):
 
 
 with gr.Blocks() as demo:
-    chatbot  = gr.Chatbot(msg_format="messages")
+    chatbot  = gr.Chatbot(type="messages")
     button = gr.Button("Get San Francisco Weather")
     button.click(generate_response, chatbot, chatbot)
 

demo/chatinterface_streaming_echo/messages_testcase.py

@@ -9,7 +9,7 @@ def slow_echo(message, history):
 
 
 
-demo = gr.ChatInterface(slow_echo, msg_format="messages")
+demo = gr.ChatInterface(slow_echo, type="messages")
 
 if __name__ == "__main__":
     demo.launch()

demo/test_chatinterface_streaming_echo/messages_testcase.py

@@ -11,7 +11,7 @@ def slow_echo(message, history):
         yield f"Run {runs} - You typed: " + message[: i + 1]
 
 
-demo = gr.ChatInterface(slow_echo, msg_format="messages").queue()
+demo = gr.ChatInterface(slow_echo, type="messages").queue()
 
 if __name__ == "__main__":
     demo.launch()