diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py index 4a359eded..2c71a2558 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_assistant_agent.py @@ -13,7 +13,7 @@ from typing import ( Sequence, ) -from autogen_core import CancellationToken, FunctionCall +from autogen_core import CancellationToken, Component, ComponentModel, FunctionCall from autogen_core.memory import Memory from autogen_core.model_context import ( ChatCompletionContext, @@ -28,6 +28,8 @@ from autogen_core.models import ( UserMessage, ) from autogen_core.tools import FunctionTool, Tool +from pydantic import BaseModel +from typing_extensions import Self from .. import EVENT_LOGGER_NAME from ..base import Handoff as HandoffBase @@ -49,7 +51,21 @@ from ._base_chat_agent import BaseChatAgent event_logger = logging.getLogger(EVENT_LOGGER_NAME) -class AssistantAgent(BaseChatAgent): +class AssistantAgentConfig(BaseModel): + """The declarative configuration for the assistant agent.""" + + name: str + model_client: ComponentModel + # tools: List[Any] | None = None # TBD + handoffs: List[HandoffBase | str] | None = None + model_context: ComponentModel | None = None + description: str + system_message: str | None = None + reflect_on_tool_use: bool + tool_call_summary_format: str + + +class AssistantAgent(BaseChatAgent, Component[AssistantAgentConfig]): """An agent that provides assistance with tool use. The :meth:`on_messages` returns a :class:`~autogen_agentchat.base.Response` @@ -229,6 +245,9 @@ class AssistantAgent(BaseChatAgent): See `o1 beta limitations `_ for more details. """ + component_config_schema = AssistantAgentConfig + component_provider_override = "autogen_agentchat.agents.AssistantAgent" + def __init__( self, name: str, @@ -462,3 +481,40 @@ class AssistantAgent(BaseChatAgent): assistant_agent_state = AssistantAgentState.model_validate(state) # Load the model context state. await self._model_context.load_state(assistant_agent_state.llm_context) + + def _to_config(self) -> AssistantAgentConfig: + """Convert the assistant agent to a declarative config.""" + + # raise an error if tools is not empty until it is implemented + # TBD : Implement serializing tools and remove this check. + if self._tools and len(self._tools) > 0: + raise NotImplementedError("Serializing tools is not implemented yet.") + + return AssistantAgentConfig( + name=self.name, + model_client=self._model_client.dump_component(), + # tools=[], # TBD + handoffs=list(self._handoffs.values()), + model_context=self._model_context.dump_component(), + description=self.description, + system_message=self._system_messages[0].content + if self._system_messages and isinstance(self._system_messages[0].content, str) + else None, + reflect_on_tool_use=self._reflect_on_tool_use, + tool_call_summary_format=self._tool_call_summary_format, + ) + + @classmethod + def _from_config(cls, config: AssistantAgentConfig) -> Self: + """Create an assistant agent from a declarative config.""" + return cls( + name=config.name, + model_client=ChatCompletionClient.load_component(config.model_client), + # tools=[], # TBD + handoffs=config.handoffs, + model_context=None, + description=config.description, + system_message=config.system_message, + reflect_on_tool_use=config.reflect_on_tool_use, + tool_call_summary_format=config.tool_call_summary_format, + ) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py index 42b7cb78a..97b9de762 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_base_chat_agent.py @@ -1,7 +1,8 @@ from abc import ABC, abstractmethod from typing import Any, AsyncGenerator, List, Mapping, Sequence -from autogen_core import CancellationToken +from autogen_core import CancellationToken, ComponentBase +from pydantic import BaseModel from ..base import ChatAgent, Response, TaskResult from ..messages import ( @@ -13,7 +14,7 @@ from ..messages import ( from ..state import BaseState -class BaseChatAgent(ChatAgent, ABC): +class BaseChatAgent(ChatAgent, ABC, ComponentBase[BaseModel]): """Base class for a chat agent. This abstract class provides a base implementation for a :class:`ChatAgent`. @@ -35,6 +36,8 @@ class BaseChatAgent(ChatAgent, ABC): This design principle must be followed when creating a new agent. """ + component_type = "agent" + def __init__(self, name: str, description: str) -> None: self._name = name if self._name.isidentifier() is False: diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py index e53d3b09a..30c444a9a 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/agents/_user_proxy_agent.py @@ -5,7 +5,9 @@ from contextvars import ContextVar from inspect import iscoroutinefunction from typing import Any, AsyncGenerator, Awaitable, Callable, ClassVar, Generator, Optional, Sequence, Union, cast -from autogen_core import CancellationToken +from autogen_core import CancellationToken, Component +from pydantic import BaseModel +from typing_extensions import Self from ..base import Response from ..messages import AgentEvent, ChatMessage, HandoffMessage, TextMessage, UserInputRequestedEvent @@ -24,7 +26,15 @@ async def cancellable_input(prompt: str, cancellation_token: Optional[Cancellati return await task -class UserProxyAgent(BaseChatAgent): +class UserProxyAgentConfig(BaseModel): + """Declarative configuration for the UserProxyAgent.""" + + name: str + description: str = "A human user" + input_func: str | None = None + + +class UserProxyAgent(BaseChatAgent, Component[UserProxyAgentConfig]): """An agent that can represent a human user through an input function. This agent can be used to represent a human user in a chat system by providing a custom input function. @@ -109,6 +119,10 @@ class UserProxyAgent(BaseChatAgent): print(f"BaseException: {e}") """ + component_type = "agent" + component_provider_override = "autogen_agentchat.agents.UserProxyAgent" + component_config_schema = UserProxyAgentConfig + class InputRequestContext: def __init__(self) -> None: raise RuntimeError( @@ -218,3 +232,11 @@ class UserProxyAgent(BaseChatAgent): async def on_reset(self, cancellation_token: Optional[CancellationToken] = None) -> None: """Reset agent state.""" pass + + def _to_config(self) -> UserProxyAgentConfig: + # TODO: Add ability to serialie input_func + return UserProxyAgentConfig(name=self.name, description=self.description, input_func=None) + + @classmethod + def _from_config(cls, config: UserProxyAgentConfig) -> Self: + return cls(name=config.name, description=config.description, input_func=None) diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py b/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py index dcefa5a04..d8a3adb96 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/base/_termination.py @@ -48,7 +48,6 @@ class TerminationCondition(ABC, ComponentBase[BaseModel]): """ component_type = "termination" - # component_config_schema = BaseModel # type: ignore @property @abstractmethod diff --git a/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py b/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py index d824815ae..e33f2fcb7 100644 --- a/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py +++ b/python/packages/autogen-agentchat/src/autogen_agentchat/conditions/_terminations.py @@ -16,7 +16,6 @@ class StopMessageTerminationConfig(BaseModel): class StopMessageTermination(TerminationCondition, Component[StopMessageTerminationConfig]): """Terminate the conversation if a StopMessage is received.""" - component_type = "termination" component_config_schema = StopMessageTerminationConfig component_provider_override = "autogen_agentchat.conditions.StopMessageTermination" @@ -58,7 +57,6 @@ class MaxMessageTermination(TerminationCondition, Component[MaxMessageTerminatio max_messages: The maximum number of messages allowed in the conversation. """ - component_type = "termination" component_config_schema = MaxMessageTerminationConfig component_provider_override = "autogen_agentchat.conditions.MaxMessageTermination" @@ -104,7 +102,6 @@ class TextMentionTermination(TerminationCondition, Component[TextMentionTerminat text: The text to look for in the messages. """ - component_type = "termination" component_config_schema = TextMentionTerminationConfig component_provider_override = "autogen_agentchat.conditions.TextMentionTermination" @@ -159,7 +156,6 @@ class TokenUsageTermination(TerminationCondition, Component[TokenUsageTerminatio ValueError: If none of max_total_token, max_prompt_token, or max_completion_token is provided. """ - component_type = "termination" component_config_schema = TokenUsageTerminationConfig component_provider_override = "autogen_agentchat.conditions.TokenUsageTermination" @@ -234,7 +230,6 @@ class HandoffTermination(TerminationCondition, Component[HandoffTerminationConfi target (str): The target of the handoff message. """ - component_type = "termination" component_config_schema = HandoffTerminationConfig component_provider_override = "autogen_agentchat.conditions.HandoffTermination" @@ -279,7 +274,6 @@ class TimeoutTermination(TerminationCondition, Component[TimeoutTerminationConfi timeout_seconds: The maximum duration in seconds before terminating the conversation. """ - component_type = "termination" component_config_schema = TimeoutTerminationConfig component_provider_override = "autogen_agentchat.conditions.TimeoutTermination" @@ -339,7 +333,6 @@ class ExternalTermination(TerminationCondition, Component[ExternalTerminationCon """ - component_type = "termination" component_config_schema = ExternalTerminationConfig component_provider_override = "autogen_agentchat.conditions.ExternalTermination" @@ -389,7 +382,6 @@ class SourceMatchTermination(TerminationCondition, Component[SourceMatchTerminat TerminatedException: If the termination condition has already been reached. """ - component_type = "termination" component_config_schema = SourceMatchTerminationConfig component_provider_override = "autogen_agentchat.conditions.SourceMatchTermination" diff --git a/python/packages/autogen-agentchat/tests/test_assistant_agent.py b/python/packages/autogen-agentchat/tests/test_assistant_agent.py index 930b4f8f7..e477ed3f1 100644 --- a/python/packages/autogen-agentchat/tests/test_assistant_agent.py +++ b/python/packages/autogen-agentchat/tests/test_assistant_agent.py @@ -592,3 +592,51 @@ async def test_run_with_memory(monkeypatch: pytest.MonkeyPatch) -> None: assert not isinstance(BadMemory(), Memory) assert isinstance(ListMemory(), Memory) + + +@pytest.mark.asyncio +async def test_assistant_agent_declarative(monkeypatch: pytest.MonkeyPatch) -> None: + model = "gpt-4o-2024-05-13" + chat_completions = [ + ChatCompletion( + id="id1", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(content="Response to message 3", role="assistant"), + ) + ], + created=0, + model=model, + object="chat.completion", + usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + mock = _MockChatCompletion(chat_completions) + monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create) + model_context = BufferedChatCompletionContext(buffer_size=2) + agent = AssistantAgent( + "test_agent", + model_client=OpenAIChatCompletionClient(model=model, api_key=""), + model_context=model_context, + ) + + agent_config = agent.dump_component() + assert agent_config.provider == "autogen_agentchat.agents.AssistantAgent" + + agent2 = AssistantAgent.load_component(agent_config) + assert agent2.name == agent.name + + agent3 = AssistantAgent( + "test_agent", + model_client=OpenAIChatCompletionClient(model=model, api_key=""), + model_context=model_context, + tools=[ + _pass_function, + _fail_function, + FunctionTool(_echo_function, description="Echo"), + ], + ) + with pytest.raises(NotImplementedError): + agent3.dump_component() diff --git a/python/packages/autogen-agentchat/tests/test_declarative_components.py b/python/packages/autogen-agentchat/tests/test_declarative_components.py index 35cf54f86..4d7ba3f38 100644 --- a/python/packages/autogen-agentchat/tests/test_declarative_components.py +++ b/python/packages/autogen-agentchat/tests/test_declarative_components.py @@ -11,6 +11,11 @@ from autogen_agentchat.conditions import ( TokenUsageTermination, ) from autogen_core import ComponentLoader, ComponentModel +from autogen_core.model_context import ( + BufferedChatCompletionContext, + HeadAndTailChatCompletionContext, + UnboundedChatCompletionContext, +) @pytest.mark.asyncio @@ -92,3 +97,35 @@ async def test_termination_declarative() -> None: # Test loading complex composition loaded_composite = ComponentLoader.load_component(composite_config) assert isinstance(loaded_composite, AndTerminationCondition) + + +@pytest.mark.asyncio +async def test_chat_completion_context_declarative() -> None: + unbounded_context = UnboundedChatCompletionContext() + buffered_context = BufferedChatCompletionContext(buffer_size=5) + head_tail_context = HeadAndTailChatCompletionContext(head_size=3, tail_size=2) + + # Test serialization + unbounded_config = unbounded_context.dump_component() + assert unbounded_config.provider == "autogen_core.model_context.UnboundedChatCompletionContext" + + buffered_config = buffered_context.dump_component() + assert buffered_config.provider == "autogen_core.model_context.BufferedChatCompletionContext" + assert buffered_config.config["buffer_size"] == 5 + + head_tail_config = head_tail_context.dump_component() + assert head_tail_config.provider == "autogen_core.model_context.HeadAndTailChatCompletionContext" + assert head_tail_config.config["head_size"] == 3 + assert head_tail_config.config["tail_size"] == 2 + + # Test deserialization + loaded_unbounded = ComponentLoader.load_component(unbounded_config, UnboundedChatCompletionContext) + assert isinstance(loaded_unbounded, UnboundedChatCompletionContext) + + loaded_buffered = ComponentLoader.load_component(buffered_config, BufferedChatCompletionContext) + + assert isinstance(loaded_buffered, BufferedChatCompletionContext) + + loaded_head_tail = ComponentLoader.load_component(head_tail_config, HeadAndTailChatCompletionContext) + + assert isinstance(loaded_head_tail, HeadAndTailChatCompletionContext) diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md index 5546417eb..e83288338 100644 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/index.md @@ -66,6 +66,18 @@ Sample code and use cases How to migrate from AutoGen 0.2.x to 0.4.x. ::: + +:::{grid-item-card} {fas}`save;pst-color-primary` Serialize Components +:link: ./serialize-components.html + +Serialize and deserialize components +::: + +:::{grid-item-card} {fas}`brain;pst-color-primary` Memory +:link: ./memory.html + +Add memory capabilities to your agents +::: :::: ```{toctree} @@ -91,8 +103,7 @@ tutorial/human-in-the-loop tutorial/termination tutorial/custom-agents tutorial/state -tutorial/declarative -tutorial/memory + ``` ```{toctree} @@ -103,6 +114,8 @@ tutorial/memory selector-group-chat swarm magentic-one +memory +serialize-components ``` ```{toctree} diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/memory.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/memory.ipynb similarity index 100% rename from python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/memory.ipynb rename to python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/memory.ipynb diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb new file mode 100644 index 000000000..5a3855f48 --- /dev/null +++ b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/serialize-components.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Serializing Components \n", + "\n", + "AutoGen provides a {py:class}`~autogen_core.Component` configuration class that defines behaviours for to serialize/deserialize component into declarative specifications. This is useful for debugging, visualizing, and even for sharing your work with others. In this notebook, we will demonstrate how to serialize multiple components to a declarative specification like a JSON file. \n", + "\n", + "\n", + "```{note}\n", + "This is work in progress\n", + "``` \n", + "\n", + "We will be implementing declarative support for the following components:\n", + "\n", + "- Termination conditions ✔️\n", + "- Tools \n", + "- Agents \n", + "- Teams \n", + "\n", + "\n", + "### Termination Condition Example \n", + "\n", + "In the example below, we will define termination conditions (a part of an agent team) in python, export this to a dictionary/json and also demonstrate how the termination condition object can be loaded from the dictionary/json. \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Config: {\"provider\":\"autogen_agentchat.base.OrTerminationCondition\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"conditions\":[{\"provider\":\"autogen_agentchat.conditions.MaxMessageTermination\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"config\":{\"max_messages\":5}},{\"provider\":\"autogen_agentchat.conditions.StopMessageTermination\",\"component_type\":\"termination\",\"version\":1,\"component_version\":1,\"config\":{}}]}}\n" + ] + } + ], + "source": [ + "from autogen_agentchat.conditions import MaxMessageTermination, StopMessageTermination\n", + "\n", + "max_termination = MaxMessageTermination(5)\n", + "stop_termination = StopMessageTermination()\n", + "\n", + "or_termination = max_termination | stop_termination\n", + "\n", + "or_term_config = or_termination.dump_component()\n", + "print(\"Config: \", or_term_config.model_dump_json())\n", + "\n", + "new_or_termination = or_termination.load_component(or_term_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Agent Example \n", + "\n", + "In the example below, we will define an agent in python, export this to a dictionary/json and also demonstrate how the agent object can be loaded from the dictionary/json." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen_agentchat.agents import AssistantAgent, UserProxyAgent\n", + "from autogen_ext.models.openai import OpenAIChatCompletionClient\n", + "\n", + "# Create an agent that uses the OpenAI GPT-4o model.\n", + "model_client = OpenAIChatCompletionClient(\n", + " model=\"gpt-4o\",\n", + " # api_key=\"YOUR_API_KEY\",\n", + ")\n", + "agent = AssistantAgent(\n", + " name=\"assistant\",\n", + " model_client=model_client,\n", + " handoffs=[\"flights_refunder\", \"user\"],\n", + " # tools=[], # serializing tools is not yet supported\n", + " system_message=\"Use tools to solve tasks.\",\n", + ")\n", + "user_proxy = UserProxyAgent(name=\"user\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"provider\":\"autogen_agentchat.agents.UserProxyAgent\",\"component_type\":\"agent\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"name\":\"user\",\"description\":\"A human user\"}}\n" + ] + } + ], + "source": [ + "user_proxy_config = user_proxy.dump_component() # dump component\n", + "print(user_proxy_config.model_dump_json())\n", + "up_new = user_proxy.load_component(user_proxy_config) # load component" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"provider\":\"autogen_agentchat.agents.AssistantAgent\",\"component_type\":\"agent\",\"version\":1,\"component_version\":1,\"description\":null,\"config\":{\"name\":\"assistant\",\"model_client\":{\"provider\":\"autogen_ext.models.openai.OpenAIChatCompletionClient\",\"component_type\":\"model\",\"version\":1,\"component_version\":1,\"config\":{\"model\":\"gpt-4o\"}},\"handoffs\":[{\"target\":\"flights_refunder\",\"description\":\"Handoff to flights_refunder.\",\"name\":\"transfer_to_flights_refunder\",\"message\":\"Transferred to flights_refunder, adopting the role of flights_refunder immediately.\"},{\"target\":\"user\",\"description\":\"Handoff to user.\",\"name\":\"transfer_to_user\",\"message\":\"Transferred to user, adopting the role of user immediately.\"}],\"model_context\":{\"provider\":\"autogen_core.model_context.UnboundedChatCompletionContext\",\"component_type\":\"chat_completion_context\",\"version\":1,\"component_version\":1,\"config\":{}},\"description\":\"An agent that provides assistance with ability to use tools.\",\"system_message\":\"Use tools to solve tasks.\",\"reflect_on_tool_use\":false,\"tool_call_summary_format\":\"{result}\"}}\n" + ] + } + ], + "source": [ + "agent_config = agent.dump_component() # dump component\n", + "print(agent_config.model_dump_json())\n", + "agent_new = agent.load_component(agent_config) # load component" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A similar approach can be used to serialize the `MultiModalWebSurfer` agent.\n", + "\n", + "```python\n", + "from autogen_ext.agents.web_surfer import MultimodalWebSurfer\n", + "\n", + "agent = MultimodalWebSurfer(\n", + " name=\"web_surfer\",\n", + " model_client=model_client,\n", + " headless=False,\n", + ")\n", + "\n", + "web_surfer_config = agent.dump_component() # dump component\n", + "print(web_surfer_config.model_dump_json())\n", + "\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb b/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb deleted file mode 100644 index 274135c11..000000000 --- a/python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/tutorial/declarative.ipynb +++ /dev/null @@ -1,119 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Declarative Components \n", - "\n", - "AutoGen provides a declarative {py:class}`~autogen_core.Component` configuration class that defines behaviours for declarative import/export. This is useful for debugging, visualizing, and even for sharing your work with others. In this notebook, we will demonstrate how to export a declarative representation of a multiagent team in the form of a JSON file. \n", - "\n", - "\n", - "```{note}\n", - "This is work in progress\n", - "``` \n", - "\n", - "We will be implementing declarative support for the following components:\n", - "\n", - "- Termination conditions ✔️\n", - "- Tools \n", - "- Agents \n", - "- Teams \n", - "\n", - "\n", - "### Termination Condition Example \n", - "\n", - "In the example below, we will define termination conditions (a part of an agent team) in python, export this to a dictionary/json and also demonstrate how the termination condition object can be loaded from the dictionary/json. \n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from autogen_agentchat.conditions import MaxMessageTermination, StopMessageTermination\n", - "\n", - "max_termination = MaxMessageTermination(5)\n", - "stop_termination = StopMessageTermination()" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "provider='autogen_agentchat.conditions.MaxMessageTermination' component_type='termination' version=1 component_version=1 description=None config={'max_messages': 5}\n" - ] - } - ], - "source": [ - "print(max_termination.dump_component())" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'provider': 'autogen_agentchat.conditions.MaxMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'description': None, 'config': {'max_messages': 5}}\n" - ] - } - ], - "source": [ - "print(max_termination.dump_component().model_dump())" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ComponentModel(provider='autogen_agentchat.base.OrTerminationCondition', component_type='termination', version=1, component_version=1, description=None, config={'conditions': [{'provider': 'autogen_agentchat.conditions.MaxMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'config': {'max_messages': 5}}, {'provider': 'autogen_agentchat.conditions.StopMessageTermination', 'component_type': 'termination', 'version': 1, 'component_version': 1, 'config': {}}]})" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "or_termination = max_termination | stop_termination\n", - "or_termination.dump_component()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py index f66197246..dcece60b1 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_buffered_chat_completion_context.py @@ -1,10 +1,19 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from ..models import FunctionExecutionResultMessage, LLMMessage from ._chat_completion_context import ChatCompletionContext -class BufferedChatCompletionContext(ChatCompletionContext): +class BufferedChatCompletionContextConfig(BaseModel): + buffer_size: int + initial_messages: List[LLMMessage] | None = None + + +class BufferedChatCompletionContext(ChatCompletionContext, Component[BufferedChatCompletionContextConfig]): """A buffered chat completion context that keeps a view of the last n messages, where n is the buffer size. The buffer size is set at initialization. @@ -13,6 +22,9 @@ class BufferedChatCompletionContext(ChatCompletionContext): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_config_schema = BufferedChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.BufferedChatCompletionContext" + def __init__(self, buffer_size: int, initial_messages: List[LLMMessage] | None = None) -> None: super().__init__(initial_messages) if buffer_size <= 0: @@ -27,3 +39,10 @@ class BufferedChatCompletionContext(ChatCompletionContext): # Remove the first message from the list. messages = messages[1:] return messages + + def _to_config(self) -> BufferedChatCompletionContextConfig: + return BufferedChatCompletionContextConfig(buffer_size=self._buffer_size, initial_messages=self._messages) + + @classmethod + def _from_config(cls, config: BufferedChatCompletionContextConfig) -> Self: + return cls(**config.model_dump()) diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py index 33b1dac7f..d2b82ec1f 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_chat_completion_context.py @@ -3,10 +3,11 @@ from typing import Any, List, Mapping from pydantic import BaseModel, Field +from .._component_config import ComponentBase from ..models import LLMMessage -class ChatCompletionContext(ABC): +class ChatCompletionContext(ABC, ComponentBase[BaseModel]): """An abstract base class for defining the interface of a chat completion context. A chat completion context lets agents store and retrieve LLM messages. It can be implemented with different recall strategies. @@ -15,6 +16,8 @@ class ChatCompletionContext(ABC): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_type = "chat_completion_context" + def __init__(self, initial_messages: List[LLMMessage] | None = None) -> None: self._messages: List[LLMMessage] = initial_messages or [] diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py index 2518f456b..a37d5927b 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_head_and_tail_chat_completion_context.py @@ -1,11 +1,21 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from .._types import FunctionCall from ..models import AssistantMessage, FunctionExecutionResultMessage, LLMMessage, UserMessage from ._chat_completion_context import ChatCompletionContext -class HeadAndTailChatCompletionContext(ChatCompletionContext): +class HeadAndTailChatCompletionContextConfig(BaseModel): + head_size: int + tail_size: int + initial_messages: List[LLMMessage] | None = None + + +class HeadAndTailChatCompletionContext(ChatCompletionContext, Component[HeadAndTailChatCompletionContextConfig]): """A chat completion context that keeps a view of the first n and last m messages, where n is the head size and m is the tail size. The head and tail sizes are set at initialization. @@ -16,6 +26,9 @@ class HeadAndTailChatCompletionContext(ChatCompletionContext): initial_messages (List[LLMMessage] | None): The initial messages. """ + component_config_schema = HeadAndTailChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.HeadAndTailChatCompletionContext" + def __init__(self, head_size: int, tail_size: int, initial_messages: List[LLMMessage] | None = None) -> None: super().__init__(initial_messages) if head_size <= 0: @@ -52,3 +65,12 @@ class HeadAndTailChatCompletionContext(ChatCompletionContext): placeholder_messages = [UserMessage(content=f"Skipped {num_skipped} messages.", source="System")] return head_messages + placeholder_messages + tail_messages + + def _to_config(self) -> HeadAndTailChatCompletionContextConfig: + return HeadAndTailChatCompletionContextConfig( + head_size=self._head_size, tail_size=self._tail_size, initial_messages=self._messages + ) + + @classmethod + def _from_config(cls, config: HeadAndTailChatCompletionContextConfig) -> Self: + return cls(head_size=config.head_size, tail_size=config.tail_size, initial_messages=config.initial_messages) diff --git a/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py b/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py index dff45bfc9..4bc26db46 100644 --- a/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py +++ b/python/packages/autogen-core/src/autogen_core/model_context/_unbounded_chat_completion_context.py @@ -1,12 +1,30 @@ from typing import List +from pydantic import BaseModel +from typing_extensions import Self + +from .._component_config import Component from ..models import LLMMessage from ._chat_completion_context import ChatCompletionContext -class UnboundedChatCompletionContext(ChatCompletionContext): +class UnboundedChatCompletionContextConfig(BaseModel): + pass + + +class UnboundedChatCompletionContext(ChatCompletionContext, Component[UnboundedChatCompletionContextConfig]): """An unbounded chat completion context that keeps a view of the all the messages.""" + component_config_schema = UnboundedChatCompletionContextConfig + component_provider_override = "autogen_core.model_context.UnboundedChatCompletionContext" + async def get_messages(self) -> List[LLMMessage]: """Get at most `buffer_size` recent messages.""" return self._messages + + def _to_config(self) -> UnboundedChatCompletionContextConfig: + return UnboundedChatCompletionContextConfig() + + @classmethod + def _from_config(cls, config: UnboundedChatCompletionContextConfig) -> Self: + return cls() diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py index d266a2086..f90dc01cd 100644 --- a/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py +++ b/python/packages/autogen-ext/src/autogen_ext/agents/web_surfer/_multimodal_web_surfer.py @@ -24,7 +24,7 @@ import PIL.Image from autogen_agentchat.agents import BaseChatAgent from autogen_agentchat.base import Response from autogen_agentchat.messages import AgentEvent, ChatMessage, MultiModalMessage, TextMessage -from autogen_core import EVENT_LOGGER_NAME, CancellationToken, FunctionCall +from autogen_core import EVENT_LOGGER_NAME, CancellationToken, Component, ComponentModel, FunctionCall from autogen_core import Image as AGImage from autogen_core.models import ( AssistantMessage, @@ -36,6 +36,8 @@ from autogen_core.models import ( ) from PIL import Image from playwright.async_api import BrowserContext, Download, Page, Playwright, async_playwright +from pydantic import BaseModel +from typing_extensions import Self from ._events import WebSurferEvent from ._prompts import WEB_SURFER_OCR_PROMPT, WEB_SURFER_QA_PROMPT, WEB_SURFER_QA_SYSTEM_MESSAGE, WEB_SURFER_TOOL_PROMPT @@ -58,7 +60,23 @@ from ._utils import message_content_to_str from .playwright_controller import PlaywrightController -class MultimodalWebSurfer(BaseChatAgent): +class MultimodalWebSurferConfig(BaseModel): + name: str + model_client: ComponentModel + downloads_folder: str | None = None + description: str | None = None + debug_dir: str | None = None + headless: bool = True + start_page: str | None = "https://www.bing.com/" + animate_actions: bool = False + to_save_screenshots: bool = False + use_ocr: bool = False + browser_channel: str | None = None + browser_data_dir: str | None = None + to_resize_viewport: bool = True + + +class MultimodalWebSurfer(BaseChatAgent, Component[MultimodalWebSurferConfig]): """ MultimodalWebSurfer is a multimodal agent that acts as a web surfer that can search the web and visit web pages. @@ -144,6 +162,10 @@ class MultimodalWebSurfer(BaseChatAgent): asyncio.run(main()) """ + component_type = "agent" + component_config_schema = MultimodalWebSurferConfig + component_provider_override = "autogen_ext.agents.web_surfer.MultimodalWebSurfer" + DEFAULT_DESCRIPTION = """ A helpful assistant with access to a web browser. Ask them to perform web searches, open pages, and interact with content (e.g., clicking links, scrolling the viewport, etc., filling in form fields, etc.). @@ -242,7 +264,8 @@ class MultimodalWebSurfer(BaseChatAgent): TOOL_SLEEP, TOOL_HOVER, ] - self.n_lines_page_text = 50 # Number of lines of text to extract from the page in the absence of OCR + # Number of lines of text to extract from the page in the absence of OCR + self.n_lines_page_text = 50 self.did_lazy_init = False # flag to check if we have initialized the browser async def _lazy_init( @@ -317,7 +340,8 @@ class MultimodalWebSurfer(BaseChatAgent): if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" - await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) + + await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) # type: ignore self.logger.info( WebSurferEvent( source=self.name, @@ -346,6 +370,7 @@ class MultimodalWebSurfer(BaseChatAgent): if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" + await self._page.screenshot(path=os.path.join(self.debug_dir, screenshot_png_name)) # type: ignore self.logger.info( WebSurferEvent( @@ -704,6 +729,7 @@ class MultimodalWebSurfer(BaseChatAgent): if self.to_save_screenshots: current_timestamp = "_" + int(time.time()).__str__() screenshot_png_name = "screenshot" + current_timestamp + ".png" + async with aiofiles.open(os.path.join(self.debug_dir, screenshot_png_name), "wb") as file: # type: ignore await file.write(new_screenshot) # type: ignore self.logger.info( @@ -861,3 +887,38 @@ class MultimodalWebSurfer(BaseChatAgent): scaled_screenshot.close() assert isinstance(response.content, str) return response.content + + def _to_config(self) -> MultimodalWebSurferConfig: + return MultimodalWebSurferConfig( + name=self.name, + model_client=self._model_client.dump_component(), + downloads_folder=self.downloads_folder, + description=self.description, + debug_dir=self.debug_dir, + headless=self.headless, + start_page=self.start_page, + animate_actions=self.animate_actions, + to_save_screenshots=self.to_save_screenshots, + use_ocr=self.use_ocr, + browser_channel=self.browser_channel, + browser_data_dir=self.browser_data_dir, + to_resize_viewport=self.to_resize_viewport, + ) + + @classmethod + def _from_config(cls, config: MultimodalWebSurferConfig) -> Self: + return cls( + name=config.name, + model_client=ChatCompletionClient.load_component(config.model_client), + downloads_folder=config.downloads_folder, + description=config.description or cls.DEFAULT_DESCRIPTION, + debug_dir=config.debug_dir, + headless=config.headless, + start_page=config.start_page or cls.DEFAULT_START_PAGE, + animate_actions=config.animate_actions, + to_save_screenshots=config.to_save_screenshots, + use_ocr=config.use_ocr, + browser_channel=config.browser_channel, + browser_data_dir=config.browser_data_dir, + to_resize_viewport=config.to_resize_viewport, + ) diff --git a/python/packages/autogen-ext/tests/test_websurfer_agent.py b/python/packages/autogen-ext/tests/test_websurfer_agent.py index d8a36e4d9..a2aa33a10 100644 --- a/python/packages/autogen-ext/tests/test_websurfer_agent.py +++ b/python/packages/autogen-ext/tests/test_websurfer_agent.py @@ -145,3 +145,38 @@ async def test_run_websurfer(monkeypatch: pytest.MonkeyPatch) -> None: ) # type: ignore url_after_sleep = agent._page.url # type: ignore assert url_after_no_tool == url_after_sleep + + +@pytest.mark.asyncio +async def test_run_websurfer_declarative(monkeypatch: pytest.MonkeyPatch) -> None: + model = "gpt-4o-2024-05-13" + chat_completions = [ + ChatCompletion( + id="id1", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage(content="Response to message 3", role="assistant"), + ) + ], + created=0, + model=model, + object="chat.completion", + usage=CompletionUsage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ), + ] + mock = _MockChatCompletion(chat_completions) + monkeypatch.setattr(AsyncCompletions, "create", mock.mock_create) + + agent = MultimodalWebSurfer( + "WebSurfer", model_client=OpenAIChatCompletionClient(model=model, api_key=""), use_ocr=False + ) + + agent_config = agent.dump_component() + assert agent_config.provider == "autogen_ext.agents.web_surfer.MultimodalWebSurfer" + assert agent_config.config["name"] == "WebSurfer" + + loaded_agent = MultimodalWebSurfer.load_component(agent_config) + assert isinstance(loaded_agent, MultimodalWebSurfer) + assert loaded_agent.name == "WebSurfer"