diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d4b84a30..2f1af9ab7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,17 +41,15 @@ jobs: pip install -e . python -c "import autogen" pip install -e. pytest mock - pip uninstall -y openai - name: Test with pytest if: matrix.python-version != '3.10' run: | - pytest test + pytest test --skip-openai - name: Coverage if: matrix.python-version == '3.10' run: | pip install -e .[test] - pip uninstall -y openai - coverage run -a -m pytest test --ignore=test/agentchat/contrib + coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai coverage xml - name: Upload coverage to Codecov if: matrix.python-version == '3.10' diff --git a/.gitignore b/.gitignore index 70bb7c9bf..03c569500 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,7 @@ test/my_tmp/* # Storage for the AgentEval output test/test_files/agenteval-in-out/out/ + +# Files created by tests +*tmp_code_* +test/agentchat/test_agent_scripts/* diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 709ca31f2..7380d6e62 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -5,6 +5,7 @@ import sys from packaging.requirements import Requirement from autogen.agentchat.contrib.agent_builder import AgentBuilder from autogen import UserProxyAgent +from conftest import skip_openai sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 @@ -20,15 +21,15 @@ try: from openai.types.completion import Completion from openai.types.completion_usage import CompletionUsage import diskcache - - OPENAI_INSTALLED = True except ImportError: - OPENAI_INSTALLED = False + skip = True +else: + skip = False or skip_openai @pytest.mark.skipif( - not OPENAI_INSTALLED, - reason="do not run when dependency is not installed", + skip, + reason="openai not installed OR requested to skip", ) def test_build(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -57,8 +58,8 @@ def test_build(): @pytest.mark.skipif( - not OPENAI_INSTALLED, - reason="do not run when dependency is not installed", + skip, + reason="openai not installed OR requested to skip", ) def test_save(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -93,8 +94,8 @@ def test_save(): @pytest.mark.skipif( - not OPENAI_INSTALLED, - reason="do not run when dependency is not installed", + skip, + reason="openai not installed OR requested to skip", ) def test_load(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -128,8 +129,8 @@ def test_load(): @pytest.mark.skipif( - not OPENAI_INSTALLED, - reason="do not run when dependency is not installed", + skip, + reason="openai not installed OR requested to skip", ) def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") diff --git a/test/agentchat/contrib/test_compressible_agent.py b/test/agentchat/contrib/test_compressible_agent.py index 089db650e..47a11ef16 100644 --- a/test/agentchat/contrib/test_compressible_agent.py +++ b/test/agentchat/contrib/test_compressible_agent.py @@ -2,6 +2,7 @@ import pytest import sys import autogen import os +from conftest import skip_openai from autogen.agentchat.contrib.compressible_agent import CompressibleAgent here = os.path.abspath(os.path.dirname(__file__)) @@ -19,15 +20,15 @@ config_list = autogen.config_list_from_json( try: import openai - - OPENAI_INSTALLED = True except ImportError: - OPENAI_INSTALLED = False + skip = True +else: + skip = False or skip_openai @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_mode_compress(): conversations = {} @@ -65,8 +66,8 @@ def test_mode_compress(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_mode_customized(): try: @@ -135,8 +136,8 @@ def test_mode_customized(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_compress_message(): assistant = CompressibleAgent( @@ -169,6 +170,10 @@ def test_compress_message(): assert is_success, "Compression failed." +@pytest.mark.skipif( + skip, + reason="do not run if dependency is not installed OR requested to skip", +) def test_mode_terminate(): assistant = CompressibleAgent( name="assistant", diff --git a/test/agentchat/contrib/test_gpt_assistant.py b/test/agentchat/contrib/test_gpt_assistant.py index f7226081e..fe92501c4 100644 --- a/test/agentchat/contrib/test_gpt_assistant.py +++ b/test/agentchat/contrib/test_gpt_assistant.py @@ -3,6 +3,7 @@ import os import sys import autogen from autogen import OpenAIWrapper +from conftest import skip_openai sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 @@ -11,10 +12,10 @@ try: import openai from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent from autogen.oai.openai_utils import retrieve_assistants_by_name - - skip_test = False except ImportError: - skip_test = True + skip = True +else: + skip = False or skip_openai config_list = autogen.config_list_from_json( OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"api_type": ["openai"]} @@ -26,8 +27,8 @@ def ask_ossinsight(question): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_gpt_assistant_chat(): ossinsight_api_schema = { @@ -73,8 +74,8 @@ def test_gpt_assistant_chat(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_get_assistant_instructions(): """ @@ -97,8 +98,8 @@ def test_get_assistant_instructions(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_gpt_assistant_instructions_overwrite(): """ @@ -142,8 +143,8 @@ def test_gpt_assistant_instructions_overwrite(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_gpt_assistant_existing_no_instructions(): """ @@ -178,8 +179,8 @@ def test_gpt_assistant_existing_no_instructions(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_get_assistant_files(): """ @@ -212,8 +213,8 @@ def test_get_assistant_files(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_assistant_retrieval(): """ @@ -283,8 +284,8 @@ def test_assistant_retrieval(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or skip_test, - reason="do not run on MacOS or windows or dependency is not installed", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip", ) def test_assistant_mismatch_retrieval(): """Test function to check if the GPTAssistantAgent can filter out the mismatch assistant""" diff --git a/test/agentchat/contrib/test_teachable_agent.py b/test/agentchat/contrib/test_teachable_agent.py index 9bb607a59..b8156296b 100644 --- a/test/agentchat/contrib/test_teachable_agent.py +++ b/test/agentchat/contrib/test_teachable_agent.py @@ -2,6 +2,7 @@ import pytest import os import sys from autogen import ConversableAgent, config_list_from_json +from conftest import skip_openai sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC # noqa: E402 @@ -12,7 +13,7 @@ try: except ImportError: skip = True else: - skip = False + skip = False or skip_openai try: from termcolor import colored diff --git a/test/agentchat/test_assistant_agent.py b/test/agentchat/test_assistant_agent.py index e0b85583b..b05a1b77b 100644 --- a/test/agentchat/test_assistant_agent.py +++ b/test/agentchat/test_assistant_agent.py @@ -2,23 +2,26 @@ import os import sys import pytest import autogen +from conftest import skip_openai from autogen.agentchat import AssistantAgent, UserProxyAgent +try: + from openai import OpenAI +except ImportError: + skip = True +else: + skip = False or skip_openai + KEY_LOC = "notebook" OAI_CONFIG_LIST = "OAI_CONFIG_LIST" here = os.path.abspath(os.path.dirname(__file__)) @pytest.mark.skipif( - sys.platform in ["darwin", "win32"], - reason="do not run on MacOS or windows", + sys.platform in ["darwin", "win32"] or skip, + reason="do not run on MacOS or windows OR openai not installed OR requested to skip", ) def test_ai_user_proxy_agent(): - try: - import openai - except ImportError: - return - conversations = {} # autogen.ChatCompletion.start_logging(conversations) @@ -57,11 +60,8 @@ def test_ai_user_proxy_agent(): print(conversations) +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") def test_gpt35(human_input_mode="NEVER", max_consecutive_auto_reply=5): - try: - import openai - except ImportError: - return config_list = autogen.config_list_from_json( OAI_CONFIG_LIST, file_location=KEY_LOC, @@ -115,12 +115,8 @@ If "Thank you" or "You\'re welcome" are said in the conversation, then say TERMI assert not isinstance(user.use_docker, bool) # None or str +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") def test_create_execute_script(human_input_mode="NEVER", max_consecutive_auto_reply=10): - try: - import openai - except ImportError: - return - config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, file_location=KEY_LOC) conversations = {} # autogen.ChatCompletion.start_logging(conversations) @@ -160,12 +156,8 @@ print('Hello world!') # autogen.ChatCompletion.stop_logging() +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10): - try: - import openai - except ImportError: - return - config_list = autogen.config_list_from_json( OAI_CONFIG_LIST, file_location=KEY_LOC, diff --git a/test/agentchat/test_async.py b/test/agentchat/test_async.py index b5f2c8fee..d31f50c0c 100644 --- a/test/agentchat/test_async.py +++ b/test/agentchat/test_async.py @@ -1,8 +1,16 @@ import pytest import asyncio import autogen +from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST +try: + from openai import OpenAI +except ImportError: + skip = True +else: + skip = False or skip_openai + def get_market_news(ind, ind_upper): data = { @@ -45,13 +53,9 @@ def get_market_news(ind, ind_upper): return feeds_summary +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") @pytest.mark.asyncio async def test_async_groupchat(): - try: - import openai - except ImportError: - return - config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC) llm_config = { @@ -91,12 +95,9 @@ async def test_async_groupchat(): assert len(user_proxy.chat_messages) > 0 +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") @pytest.mark.asyncio async def test_stream(): - try: - import openai - except ImportError: - return config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC) data = asyncio.Future() diff --git a/test/agentchat/test_async_get_human_input.py b/test/agentchat/test_async_get_human_input.py index 2daca19f2..f9de9f1c2 100644 --- a/test/agentchat/test_async_get_human_input.py +++ b/test/agentchat/test_async_get_human_input.py @@ -1,15 +1,20 @@ import asyncio import autogen import pytest +from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST +try: + from openai import OpenAI +except ImportError: + skip = True +else: + skip = False or skip_openai + +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") @pytest.mark.asyncio async def test_async_get_human_input(): - try: - import openai - except ImportError: - return config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC) # create an AssistantAgent instance named "assistant" diff --git a/test/agentchat/test_conversable_agent.py b/test/agentchat/test_conversable_agent.py index 3a23fd8fd..87868b05d 100644 --- a/test/agentchat/test_conversable_agent.py +++ b/test/agentchat/test_conversable_agent.py @@ -2,6 +2,7 @@ import copy from typing import Any, Callable, Dict, Literal import pytest +from unittest.mock import patch from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -421,7 +422,7 @@ def test__wrap_function_sync(): else: raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}") - agent = ConversableAgent(name="agent", llm_config={}) + agent = ConversableAgent(name="agent", llm_config=False) @agent._wrap_function def currency_calculator( @@ -457,7 +458,7 @@ async def test__wrap_function_async(): else: raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}") - agent = ConversableAgent(name="agent", llm_config={}) + agent = ConversableAgent(name="agent", llm_config=False) @agent._wrap_function async def currency_calculator( diff --git a/test/agentchat/test_function_call.py b/test/agentchat/test_function_call.py index cfe606b08..54f1f6dff 100644 --- a/test/agentchat/test_function_call.py +++ b/test/agentchat/test_function_call.py @@ -1,17 +1,21 @@ -try: - from openai import OpenAI -except ImportError: - OpenAI = None import pytest import asyncio import json import autogen +from conftest import skip_openai from autogen.math_utils import eval_math_responses from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import sys +try: + from openai import OpenAI +except ImportError: + skip = True +else: + skip = False or skip_openai -@pytest.mark.skipif(OpenAI is None, reason="openai>=1 not installed") + +@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip") def test_eval_math_responses(): config_list = autogen.config_list_from_models( KEY_LOC, exclude="aoai", model_list=["gpt-4-0613", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k"] @@ -190,8 +194,8 @@ async def test_a_execute_function(): @pytest.mark.skipif( - not OpenAI or not sys.version.startswith("3.10"), - reason="do not run if openai is not installed or py!=3.10", + skip or not sys.version.startswith("3.10"), + reason="do not run if openai is not installed OR reeusted to skip OR py!=3.10", ) def test_update_function(): config_list_gpt4 = autogen.config_list_from_json( diff --git a/test/agentchat/test_function_call_groupchat.py b/test/agentchat/test_function_call_groupchat.py index d8fdf30d3..02fe14a3e 100644 --- a/test/agentchat/test_function_call_groupchat.py +++ b/test/agentchat/test_function_call_groupchat.py @@ -1,6 +1,7 @@ import autogen import pytest import sys +from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST try: @@ -8,7 +9,7 @@ try: except ImportError: skip = True else: - skip = False + skip = False or skip_openai @pytest.mark.skipif( diff --git a/test/agentchat/test_math_user_proxy_agent.py b/test/agentchat/test_math_user_proxy_agent.py index bb825ddfb..787a63f04 100644 --- a/test/agentchat/test_math_user_proxy_agent.py +++ b/test/agentchat/test_math_user_proxy_agent.py @@ -6,6 +6,7 @@ from autogen.agentchat.contrib.math_user_proxy_agent import ( _remove_print, _add_print_to_last_line, ) +from conftest import skip_openai from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST try: @@ -13,7 +14,7 @@ try: except ImportError: skip = True else: - skip = False + skip = False or skip_openai @pytest.mark.skipif( diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..8cf1762b4 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,16 @@ +import pytest + +skip_openai = False + + +# Registers command-line option '--skip-openai' via pytest hook. +# When this flag is set, it indicates that tests requiring OpenAI should be skipped. +def pytest_addoption(parser): + parser.addoption("--skip-openai", action="store_true", help="Skip all tests that require openai") + + +# pytest hook implementation extracting the '--skip-openai' command line arg and exposing it globally +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + global skip_openai + skip_openai = config.getoption("--skip-openai", False) diff --git a/test/oai/test_client.py b/test/oai/test_client.py index 4ac9374c3..749c272ea 100644 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -1,18 +1,19 @@ import pytest from autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai +from conftest import skip_openai TOOL_ENABLED = False try: from openai import OpenAI - from openai.types.chat.chat_completion import ChatCompletionMessage -except ImportError: - skip = True -else: - skip = False import openai if openai.__version__ >= "1.1.0": TOOL_ENABLED = True + from openai.types.chat.chat_completion import ChatCompletionMessage +except ImportError: + skip = True +else: + skip = False or skip_openai KEY_LOC = "notebook" OAI_CONFIG_LIST = "OAI_CONFIG_LIST" @@ -35,7 +36,7 @@ def test_aoai_chat_completion(): print(client.extract_text_or_completion_object(response)) -@pytest.mark.skipif(skip and not TOOL_ENABLED, reason="openai>=1.1.0 not installed") +@pytest.mark.skipif(skip or not TOOL_ENABLED, reason="openai>=1.1.0 not installed") def test_oai_tool_calling_extraction(): config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, diff --git a/test/oai/test_client_stream.py b/test/oai/test_client_stream.py index 613770964..c0ff76071 100644 --- a/test/oai/test_client_stream.py +++ b/test/oai/test_client_stream.py @@ -1,12 +1,13 @@ import pytest from autogen import OpenAIWrapper, config_list_from_json, config_list_openai_aoai +from conftest import skip_openai try: from openai import OpenAI except ImportError: skip = True else: - skip = False + skip = False or skip_openai KEY_LOC = "notebook" OAI_CONFIG_LIST = "OAI_CONFIG_LIST" diff --git a/test/test_notebook.py b/test/test_notebook.py index fc10f4afd..3abdbc230 100644 --- a/test/test_notebook.py +++ b/test/test_notebook.py @@ -1,13 +1,14 @@ import sys import os import pytest +from conftest import skip_openai try: import openai - - skip = False except ImportError: skip = True +else: + skip = False or skip_openai here = os.path.abspath(os.path.dirname(__file__)) diff --git a/test/test_retrieve_utils.py b/test/test_retrieve_utils.py index 013228e47..ba7485f5f 100644 --- a/test/test_retrieve_utils.py +++ b/test/test_retrieve_utils.py @@ -1,6 +1,8 @@ """ Unit test for retrieve_utils.py """ +import pytest + try: import chromadb from autogen.retrieve_utils import ( @@ -18,8 +20,6 @@ except ImportError: else: skip = False import os -import sys -import pytest try: from unstructured.partition.auto import partition diff --git a/website/docs/Contribute.md b/website/docs/Contribute.md index 4ed0ed755..3b13d2f8e 100644 --- a/website/docs/Contribute.md +++ b/website/docs/Contribute.md @@ -120,22 +120,20 @@ Tests are automatically run via GitHub actions. There are two workflows: 1. [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) 1. [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml) -The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in one python version only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml). +The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml). #### Run non-OpenAI tests To run the subset of the tests not depending on `openai` (and not calling LLMs)): -- Install `pytest` -- Remove `openai` library, this is required to skip the tests which check for `openai` presence ([`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19)) -- Run the tests from the `test` folder. Make sure they all pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass -- Reinstall `openai` - +- Install pytest: ``` pip install pytest -pip uninstall -y openai -pytest test -pip install openai ``` +- Run the tests from the `test` folder using the `--skip-openai` flag. +``` +pytest test --skip-openai +``` +- Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass ### Coverage