autogen/test/openai/test_notebook.py
Qingyun Wu 45641000c0
Adding a test function for OpenAI completion in flaml (#951)
* improve max_valid_n and doc

* Update README.md

Co-authored-by: Li Jiang <lijiang1@microsoft.com>

* add support for chatgpt

* notebook

* newline at end of file

* chatgpt notebook

* ChatGPT in Azure

* doc

* math

* warning, timeout, log file name

* handle import error

* doc update; default value

* paper

* doc

* docstr

* eval_func

* add a test func in completion

* update notebook

* update math notebook

* improve notebok

* lint and handle exception

* flake8

* exception in test

* add agg_method

* NameError

* refactor

* Update flaml/integrations/oai/completion.py

Co-authored-by: Chi Wang <wang.chi@microsoft.com>

* Update flaml/integrations/oai/completion.py

Co-authored-by: Chi Wang <wang.chi@microsoft.com>

* add example

* merge files from oai_eval_test

* Revert "merge files from oai_eval_test"
This reverts commit 1e6a550f913bb94df6e9680934ccb7175d00702e.

* merge

* save results to notebook_output

* update version and cache

* update doc

* save nb cell results to file

* fix typo in model name

* code improvements

* improve docstr

* docstr

* docstr on the Returns of test

---------

Co-authored-by: Chi Wang <wang.chi@microsoft.com>
Co-authored-by: Li Jiang <lijiang1@microsoft.com>
Co-authored-by: Susan Xueqing Liu <liususan091219@users.noreply.github.com>
2023-04-02 16:14:11 +00:00

63 lines
1.9 KiB
Python

import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert.preprocessors import CellExecutionError
import os
import pytest
try:
import openai
skip = False
except ImportError:
skip = True
here = os.path.abspath(os.path.dirname(__file__))
def run_notebook(input_nb, output_nb="executed_openai_notebook.ipynb", save=False):
try:
file_path = os.path.join(here, os.pardir, os.pardir, "notebook", input_nb)
with open(file_path) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=3600, kernel_name="python3")
ep.preprocess(nb, {"metadata": {"path": here}})
output_file_name = "executed_openai_notebook_output.txt"
output_file = os.path.join(here, output_file_name)
with open(output_file, "a") as f:
for cell in nb.cells:
if cell.cell_type == "code" and "outputs" in cell:
for output in cell.outputs:
if "text" in output:
f.write(output["text"].strip() + "\n")
elif "data" in output and "text/plain" in output["data"]:
f.write(output["data"]["text/plain"].strip() + "\n")
except CellExecutionError:
raise
finally:
if save:
with open(os.path.join(here, output_nb), "w", encoding="utf-8") as f:
nbformat.write(nb, f)
@pytest.mark.skipif(
skip,
reason="do not run openai test if openai is not installed",
)
def test_integrate_openai(save=False):
run_notebook("integrate_openai.ipynb", save=save)
@pytest.mark.skipif(
skip,
reason="do not run openai test if openai is not installed",
)
def test_integrate_chatgpt(save=False):
run_notebook("integrate_chatgpt.ipynb", save=save)
if __name__ == "__main__":
test_integrate_chatgpt(save=True)
test_integrate_openai(save=True)