diff --git a/python/packages/autogen-agentchat/pyproject.toml b/python/packages/autogen-agentchat/pyproject.toml index b1c52a0bb..b8b694d96 100644 --- a/python/packages/autogen-agentchat/pyproject.toml +++ b/python/packages/autogen-agentchat/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ ] dependencies = [ "autogen-core==0.4.0.dev13", + "aioconsole>=0.8.1" ] [tool.ruff] diff --git a/python/packages/autogen-core/pyproject.toml b/python/packages/autogen-core/pyproject.toml index 41f36ef59..9c15908f3 100644 --- a/python/packages/autogen-core/pyproject.toml +++ b/python/packages/autogen-core/pyproject.toml @@ -15,24 +15,20 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "openai>=1.3", "pillow>=11.0.0", - "aioconsole>=0.8.1", - "aiohttp>=3.10.10", - "typing-extensions", + "typing-extensions>=4.0.0", "pydantic<3.0.0,>=2.10.0", "protobuf~=4.25.1", - "tiktoken>=0.8.0", - "opentelemetry-api~=1.27.0", - "asyncio_atexit", + "opentelemetry-api>=1.27.0", "jsonref~=1.1.0", ] [dependency-groups] dev = [ - "autogen_test_utils", "aiofiles", + "asyncio_atexit", + "autogen_test_utils", "azure-identity", "chess", "colorama", @@ -46,6 +42,7 @@ dev = [ "llama-index", "markdownify", "nbqa", + "opentelemetry-sdk>=1.27.0", "pip", "polars", "python-dotenv", @@ -55,12 +52,11 @@ dev = [ "textual-imageview", "textual", "types-aiofiles", + "types-docker", "types-pillow", "types-protobuf", "types-requests", - "types-docker", "wikipedia", - "opentelemetry-sdk>=1.27.0", # Documentation "myst-nb==1.1.2", diff --git a/python/packages/autogen-core/src/autogen_core/_image.py b/python/packages/autogen-core/src/autogen_core/_image.py index 1b360ca46..e24dfaa6b 100644 --- a/python/packages/autogen-core/src/autogen_core/_image.py +++ b/python/packages/autogen-core/src/autogen_core/_image.py @@ -4,10 +4,8 @@ import base64 import re from io import BytesIO from pathlib import Path -from typing import Any, cast +from typing import Any, Dict, cast -import aiohttp -from openai.types.chat import ChatCompletionContentPartImageParam from PIL import Image as PILImage from pydantic import GetCoreSchemaHandler, ValidationInfo from pydantic_core import core_schema @@ -15,6 +13,32 @@ from typing_extensions import Literal class Image: + """Represents an image. + + + Example: + + Loading an image from a URL: + + .. code-block:: python + + from autogen_core import Image + from PIL import Image as PILImage + import aiohttp + import asyncio + + + async def from_url(url: str) -> Image: + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + content = await response.read() + return Image.from_pil(PILImage.open(content)) + + + image = asyncio.run(from_url("https://example.com/image")) + + """ + def __init__(self, image: PILImage.Image): self.image: PILImage.Image = image.convert("RGB") @@ -31,13 +55,6 @@ class Image: base64_data = re.sub(r"data:image/(?:png|jpeg);base64,", "", uri) return cls.from_base64(base64_data) - @classmethod - async def from_url(cls, url: str) -> Image: - async with aiohttp.ClientSession() as session: - async with session.get(url) as response: - content = await response.read() - return cls(PILImage.open(content)) - @classmethod def from_base64(cls, base64_str: str) -> Image: return cls(PILImage.open(BytesIO(base64.b64decode(base64_str)))) @@ -60,7 +77,9 @@ class Image: def data_uri(self) -> str: return _convert_base64_to_data_uri(self.to_base64()) - def to_openai_format(self, detail: Literal["auto", "low", "high"] = "auto") -> ChatCompletionContentPartImageParam: + # Returns openai.types.chat.ChatCompletionContentPartImageParam, which is a TypedDict + # We don't use the explicit type annotation so that we can avoid a dependency on the OpenAI Python SDK in this package. + def to_openai_format(self, detail: Literal["auto", "low", "high"] = "auto") -> Dict[str, Any]: return {"type": "image_url", "image_url": {"url": self.data_uri, "detail": detail}} @classmethod diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py index e3117a5d9..2975d19b6 100644 --- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py +++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py @@ -51,6 +51,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI from openai.types.chat import ( ChatCompletion, ChatCompletionAssistantMessageParam, + ChatCompletionContentPartImageParam, ChatCompletionContentPartParam, ChatCompletionContentPartTextParam, ChatCompletionMessageParam, @@ -154,7 +155,7 @@ def user_message_to_oai(message: UserMessage) -> ChatCompletionUserMessageParam: elif isinstance(part, Image): # TODO: support url based images # TODO: support specifying details - parts.append(part.to_openai_format()) + parts.append(cast(ChatCompletionContentPartImageParam, part.to_openai_format())) else: raise ValueError(f"Unknown content type: {part}") return ChatCompletionUserMessageParam( diff --git a/python/uv.lock b/python/uv.lock index 6aa63d6ee..84913dfed 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -336,33 +336,33 @@ name = "autogen-agentchat" version = "0.4.0.dev13" source = { editable = "packages/autogen-agentchat" } dependencies = [ + { name = "aioconsole" }, { name = "autogen-core" }, ] [package.metadata] -requires-dist = [{ name = "autogen-core", editable = "packages/autogen-core" }] +requires-dist = [ + { name = "aioconsole", specifier = ">=0.8.1" }, + { name = "autogen-core", editable = "packages/autogen-core" }, +] [[package]] name = "autogen-core" version = "0.4.0.dev13" source = { editable = "packages/autogen-core" } dependencies = [ - { name = "aioconsole" }, - { name = "aiohttp" }, - { name = "asyncio-atexit" }, { name = "jsonref" }, - { name = "openai" }, { name = "opentelemetry-api" }, { name = "pillow" }, { name = "protobuf" }, { name = "pydantic" }, - { name = "tiktoken" }, { name = "typing-extensions" }, ] [package.dev-dependencies] dev = [ { name = "aiofiles" }, + { name = "asyncio-atexit" }, { name = "autodoc-pydantic" }, { name = "autogen-ext" }, { name = "autogen-test-utils" }, @@ -407,22 +407,18 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aioconsole", specifier = ">=0.8.1" }, - { name = "aiohttp", specifier = ">=3.10.10" }, - { name = "asyncio-atexit" }, { name = "jsonref", specifier = "~=1.1.0" }, - { name = "openai", specifier = ">=1.3" }, - { name = "opentelemetry-api", specifier = "~=1.27.0" }, + { name = "opentelemetry-api", specifier = ">=1.27.0" }, { name = "pillow", specifier = ">=11.0.0" }, { name = "protobuf", specifier = "~=4.25.1" }, { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, - { name = "tiktoken", specifier = ">=0.8.0" }, - { name = "typing-extensions" }, + { name = "typing-extensions", specifier = ">=4.0.0" }, ] [package.metadata.requires-dev] dev = [ { name = "aiofiles" }, + { name = "asyncio-atexit" }, { name = "autodoc-pydantic", specifier = "~=2.2" }, { name = "autogen-ext", editable = "packages/autogen-ext" }, { name = "autogen-test-utils", editable = "packages/autogen-test-utils" },