mirror of
https://github.com/microsoft/autogen.git
synced 2025-12-28 15:38:53 +00:00
doc: improve tool guide in Core API doc (#5546)
- Add a section on how to use model client with tools - Replace the obsecure pattern of using ToolAgent with a simpler implementation of calling tool within the agent itself.
This commit is contained in:
Eric Zhu
GitHub
parent
acd7e86430
commit
a1234bc658
@ -1,48 +0,0 @@
|
||||
<mxfile host="app.diagrams.net" agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0" version="24.7.5">
|
||||
<diagram name="Page-1" id="2U5l3ylZQluw78bWm4Ui">
|
||||
<mxGraphModel dx="970" dy="1091" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-10" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0;exitDx=0;exitDy=0;entryX=0;entryY=1;entryDx=0;entryDy=0;curved=1;" parent="1" source="HfMixxTfmlLCSILu5LXk-1" target="HfMixxTfmlLCSILu5LXk-2" edge="1">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="240" y="370" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-1" value="ToolAgent:handle_function_call" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" parent="1" vertex="1">
|
||||
<mxGeometry x="265" y="370" width="190" height="40" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-8" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0;entryDx=0;entryDy=0;curved=1;" parent="1" source="HfMixxTfmlLCSILu5LXk-2" target="HfMixxTfmlLCSILu5LXk-1" edge="1">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<Array as="points">
|
||||
<mxPoint x="500" y="300" />
|
||||
<mxPoint x="500" y="370" />
|
||||
</Array>
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="HfMixxTfmlLCSILu5LXk-2" edge="1">
|
||||
<mxGeometry relative="1" as="geometry">
|
||||
<mxPoint x="600" y="270" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-2" value="ToolUseAgent:handle_user_message<div>model_client.create</div>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" parent="1" vertex="1">
|
||||
<mxGeometry x="240" y="240" width="240" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="HfMixxTfmlLCSILu5LXk-3" target="HfMixxTfmlLCSILu5LXk-2" edge="1">
|
||||
<mxGeometry relative="1" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-3" value="send message" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
|
||||
<mxGeometry x="120" y="255" width="60" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-12" value="model response is tool call" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
|
||||
<mxGeometry x="510" y="330" width="90" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
<mxCell id="HfMixxTfmlLCSILu5LXk-16" value="model response is text" style="text;html=1;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" parent="1" vertex="1">
|
||||
<mxGeometry x="500" y="225" width="90" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 82 KiB |
@ -66,10 +66,14 @@
|
||||
"The {py:class}`~autogen_ext.code_executors.docker.DockerCommandLineCodeExecutor`\n",
|
||||
"class is a built-in code executor that runs Python code snippets in a subprocess\n",
|
||||
"in the command line environment of a docker container.\n",
|
||||
"The {py:class}`~autogen_core.tools.PythonCodeExecutionTool` class wraps the code executor\n",
|
||||
"The {py:class}`~autogen_ext.tools.code_execution.PythonCodeExecutionTool` class wraps the code executor\n",
|
||||
"and provides a simple interface to execute Python code snippets.\n",
|
||||
"\n",
|
||||
"Other built-in tools will be added in the future."
|
||||
"Examples of other built-in tools\n",
|
||||
"- {py:class}`~autogen_ext.tools.graphrag.LocalSearchTool` and {py:class}`~autogen_ext.tools.graphrag.GlobalSearchTool` for using [GraphRAG](https://github.com/microsoft/graphrag).\n",
|
||||
"- {py:class}`~autogen_ext.tools.mcp.mcp_server_tools` for using [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) servers as tools.\n",
|
||||
"- {py:class}`~autogen_ext.tools.http.HttpTool` for making HTTP requests to REST APIs.\n",
|
||||
"- {py:class}`~autogen_ext.tools.langchain.LangChainToolAdapter` for using LangChain tools."
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -92,14 +96,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"34.26925801998722\n"
|
||||
"36.63801673457121\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -131,25 +135,146 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Tool-Equipped Agent\n",
|
||||
"## Calling Tools with Model Clients\n",
|
||||
"\n",
|
||||
"To use tools with an agent, you can use {py:class}`~autogen_core.tool_agent.ToolAgent`,\n",
|
||||
"by using it in a composition pattern.\n",
|
||||
"Here is an example tool-use agent that uses {py:class}`~autogen_core.tool_agent.ToolAgent`\n",
|
||||
"as an inner agent for executing tools."
|
||||
"Model clients can generate tool calls when they are provided with a list of tools.\n",
|
||||
"\n",
|
||||
"Here is an example of how to use the {py:class}`~autogen_core.tools.FunctionTool` class\n",
|
||||
"with a {py:class}`~autogen_ext.models.openai.OpenAIChatCompletionClient`.\n",
|
||||
"Other model client classes can be used in a similar way. See [Model Clients](./model-clients.ipynb)\n",
|
||||
"for more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[FunctionCall(id='call_tpJ5J1Xoxi84Sw4v0scH0qBM', arguments='{\"ticker\":\"AAPL\",\"date\":\"2021/01/01\"}', name='get_stock_price')]"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"from autogen_core.models import AssistantMessage, FunctionExecutionResult, FunctionExecutionResultMessage, UserMessage\n",
|
||||
"from autogen_ext.models.openai import OpenAIChatCompletionClient\n",
|
||||
"\n",
|
||||
"# Create the OpenAI chat completion client. Using OPENAI_API_KEY from environment variable.\n",
|
||||
"client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
|
||||
"\n",
|
||||
"# Create a user message.\n",
|
||||
"user_message = UserMessage(content=\"What is the stock price of AAPL on 2021/01/01?\", source=\"user\")\n",
|
||||
"\n",
|
||||
"# Run the chat completion with the stock_price_tool defined above.\n",
|
||||
"cancellation_token = CancellationToken()\n",
|
||||
"create_result = await client.create(\n",
|
||||
" messages=[user_message], tools=[stock_price_tool], cancellation_token=cancellation_token\n",
|
||||
")\n",
|
||||
"create_result.content"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The result is a list of {py:class}`~autogen_core.FunctionCall` objects, which can be\n",
|
||||
"used to run the corresponding tools."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'32.381250753393104'"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"arguments = json.loads(create_result.content[0].arguments) # type: ignore\n",
|
||||
"tool_result = await stock_price_tool.run_json(arguments, cancellation_token)\n",
|
||||
"tool_result_str = stock_price_tool.return_value_as_string(tool_result)\n",
|
||||
"tool_result_str"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now you can make another model client call to have the model generate a reflection\n",
|
||||
"on the result of the tool execution.\n",
|
||||
"\n",
|
||||
"The result of the tool call is wrapped in a {py:class}`~autogen_core.models.FunctionExecutionResult`\n",
|
||||
"object, which contains the result of the tool execution and the ID of the tool that was called.\n",
|
||||
"The model client can use this information to generate a reflection on the result of the tool execution."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The stock price of AAPL (Apple Inc.) on January 1, 2021, was approximately $32.38.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Create a function execution result\n",
|
||||
"exec_result = FunctionExecutionResult(call_id=create_result.content[0].id, content=tool_result_str, is_error=False) # type: ignore\n",
|
||||
"\n",
|
||||
"# Make another chat completion with the history and function execution result message.\n",
|
||||
"messages = [\n",
|
||||
" user_message,\n",
|
||||
" AssistantMessage(content=create_result.content, source=\"assistant\"), # assistant message with tool call\n",
|
||||
" FunctionExecutionResultMessage(content=[exec_result]), # function execution result message\n",
|
||||
"]\n",
|
||||
"create_result = await client.create(messages=messages, cancellation_token=cancellation_token) # type: ignore\n",
|
||||
"print(create_result.content)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Tool-Equipped Agent\n",
|
||||
"\n",
|
||||
"Putting the model client and the tools together, you can create a tool-equipped agent\n",
|
||||
"that can use tools to perform actions, and reflect on the results of those actions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import asyncio\n",
|
||||
"import json\n",
|
||||
"from dataclasses import dataclass\n",
|
||||
"from typing import List\n",
|
||||
"\n",
|
||||
"from autogen_core import (\n",
|
||||
" AgentId,\n",
|
||||
" FunctionCall,\n",
|
||||
" MessageContext,\n",
|
||||
" RoutedAgent,\n",
|
||||
" SingleThreadedAgentRuntime,\n",
|
||||
@ -161,8 +286,7 @@
|
||||
" SystemMessage,\n",
|
||||
" UserMessage,\n",
|
||||
")\n",
|
||||
"from autogen_core.tool_agent import ToolAgent, tool_agent_caller_loop\n",
|
||||
"from autogen_core.tools import FunctionTool, Tool, ToolSchema\n",
|
||||
"from autogen_core.tools import FunctionTool, Tool\n",
|
||||
"from autogen_ext.models.openai import OpenAIChatCompletionClient\n",
|
||||
"\n",
|
||||
"\n",
|
||||
@ -172,60 +296,83 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"class ToolUseAgent(RoutedAgent):\n",
|
||||
" def __init__(self, model_client: ChatCompletionClient, tool_schema: List[ToolSchema], tool_agent_type: str) -> None:\n",
|
||||
" def __init__(self, model_client: ChatCompletionClient, tool_schema: List[Tool]) -> None:\n",
|
||||
" super().__init__(\"An agent with tools\")\n",
|
||||
" self._system_messages: List[LLMMessage] = [SystemMessage(content=\"You are a helpful AI assistant.\")]\n",
|
||||
" self._model_client = model_client\n",
|
||||
" self._tool_schema = tool_schema\n",
|
||||
" self._tool_agent_id = AgentId(tool_agent_type, self.id.key)\n",
|
||||
" self._tools = tools\n",
|
||||
"\n",
|
||||
" @message_handler\n",
|
||||
" async def handle_user_message(self, message: Message, ctx: MessageContext) -> Message:\n",
|
||||
" # Create a session of messages.\n",
|
||||
" session: List[LLMMessage] = self._system_messages + [UserMessage(content=message.content, source=\"user\")]\n",
|
||||
" # Run the caller loop to handle tool calls.\n",
|
||||
" messages = await tool_agent_caller_loop(\n",
|
||||
" self,\n",
|
||||
" tool_agent_id=self._tool_agent_id,\n",
|
||||
" model_client=self._model_client,\n",
|
||||
" input_messages=session,\n",
|
||||
" tool_schema=self._tool_schema,\n",
|
||||
"\n",
|
||||
" # Run the chat completion with the tools.\n",
|
||||
" create_result = await self._model_client.create(\n",
|
||||
" messages=session,\n",
|
||||
" tools=self._tools,\n",
|
||||
" cancellation_token=ctx.cancellation_token,\n",
|
||||
" )\n",
|
||||
" # Return the final response.\n",
|
||||
" assert isinstance(messages[-1].content, str)\n",
|
||||
" return Message(content=messages[-1].content)"
|
||||
"\n",
|
||||
" # If there are no tool calls, return the result.\n",
|
||||
" if isinstance(create_result.content, str):\n",
|
||||
" return Message(content=create_result.content)\n",
|
||||
" assert isinstance(create_result.content, list) and all(\n",
|
||||
" isinstance(call, FunctionCall) for call in create_result.content\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Add the first model create result to the session.\n",
|
||||
" session.append(AssistantMessage(content=create_result.content, source=\"assistant\"))\n",
|
||||
"\n",
|
||||
" # Execute the tool calls.\n",
|
||||
" results = await asyncio.gather(\n",
|
||||
" *[self._execute_tool_call(call, ctx.cancellation_token) for call in create_result.content]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Add the function execution results to the session.\n",
|
||||
" session.append(FunctionExecutionResultMessage(content=results))\n",
|
||||
"\n",
|
||||
" # Run the chat completion again to reflect on the history and function execution results.\n",
|
||||
" create_result = await self._model_client.create(\n",
|
||||
" messages=session,\n",
|
||||
" cancellation_token=ctx.cancellation_token,\n",
|
||||
" )\n",
|
||||
" assert isinstance(create_result.content, str)\n",
|
||||
"\n",
|
||||
" # Return the result as a message.\n",
|
||||
" return Message(content=create_result.content)\n",
|
||||
"\n",
|
||||
" async def _execute_tool_call(\n",
|
||||
" self, call: FunctionCall, cancellation_token: CancellationToken\n",
|
||||
" ) -> FunctionExecutionResult:\n",
|
||||
" # Find the tool by name.\n",
|
||||
" tool = next((tool for tool in self._tools if tool.name == call.name), None)\n",
|
||||
" assert tool is not None\n",
|
||||
"\n",
|
||||
" # Run the tool and capture the result.\n",
|
||||
" try:\n",
|
||||
" arguments = json.loads(call.arguments)\n",
|
||||
" result = await tool.run_json(arguments, cancellation_token)\n",
|
||||
" return FunctionExecutionResult(call_id=call.id, content=tool.return_value_as_string(result), is_error=False)\n",
|
||||
" except Exception as e:\n",
|
||||
" return FunctionExecutionResult(call_id=call.id, content=str(e), is_error=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The `ToolUseAgent` class uses a convenience function {py:meth}`~autogen_core.tool_agent.tool_agent_caller_loop`, \n",
|
||||
"to handle the interaction between the model and the tool agent.\n",
|
||||
"The core idea can be described using a simple control flow graph:\n",
|
||||
"When handling a user message, the `ToolUseAgent` class first use the model client\n",
|
||||
"to generate a list of function calls to the tools, and then run the tools\n",
|
||||
"and generate a reflection on the results of the tool execution.\n",
|
||||
"The reflection is then returned to the user as the agent's response.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The `ToolUseAgent`'s `handle_user_message` handler handles messages from the user,\n",
|
||||
"and determines whether the model has generated a tool call.\n",
|
||||
"If the model has generated tool calls, then the handler sends a function call\n",
|
||||
"message to the {py:class}`~autogen_core.tool_agent.ToolAgent` agent\n",
|
||||
"to execute the tools,\n",
|
||||
"and then queries the model again with the results of the tool calls.\n",
|
||||
"This process continues until the model stops generating tool calls,\n",
|
||||
"at which point the final response is returned to the user.\n",
|
||||
"\n",
|
||||
"By having the tool execution logic in a separate agent,\n",
|
||||
"we expose the model-tool interactions to the agent runtime as messages, so the tool executions\n",
|
||||
"can be observed externally and intercepted if necessary.\n",
|
||||
"\n",
|
||||
"To run the agent, we need to create a runtime and register the agent."
|
||||
"To run the agent, let's create a runtime and register the agent with the runtime."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@ -234,7 +381,7 @@
|
||||
"AgentType(type='tool_use_agent')"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -245,12 +392,12 @@
|
||||
"# Create the tools.\n",
|
||||
"tools: List[Tool] = [FunctionTool(get_stock_price, description=\"Get the stock price.\")]\n",
|
||||
"# Register the agents.\n",
|
||||
"await ToolAgent.register(runtime, \"tool_executor_agent\", lambda: ToolAgent(\"tool executor agent\", tools))\n",
|
||||
"await ToolUseAgent.register(\n",
|
||||
" runtime,\n",
|
||||
" \"tool_use_agent\",\n",
|
||||
" lambda: ToolUseAgent(\n",
|
||||
" OpenAIChatCompletionClient(model=\"gpt-4o-mini\"), [tool.schema for tool in tools], \"tool_executor_agent\"\n",
|
||||
" OpenAIChatCompletionClient(model=\"gpt-4o-mini\"),\n",
|
||||
" tools,\n",
|
||||
" ),\n",
|
||||
")"
|
||||
]
|
||||
@ -259,21 +406,21 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This example uses the {py:class}`autogen_core.models.OpenAIChatCompletionClient`,\n",
|
||||
"This example uses the {py:class}`~autogen_ext.models.openai.OpenAIChatCompletionClient`,\n",
|
||||
"for Azure OpenAI and other clients, see [Model Clients](./model-clients.ipynb).\n",
|
||||
"Let's test the agent with a question about stock price."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The stock price of NVDA (NVIDIA Corporation) on June 1, 2024, was approximately $27.41.\n"
|
||||
"The stock price of NVIDIA (NVDA) on June 1, 2024, was approximately $140.05.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user