mirror of
https://github.com/microsoft/autogen.git
synced 2025-08-11 18:21:31 +00:00
cd199c7ab7
* Initial commit of AutoGenBench * wording * typo * pre-commit reformulation * Updated README to point to contributor's guide earlier. * Simplified the description of the JSON format. * Added print statements to indicate when run.sh and scenario.py are starting. * Added SocietyOfMind scenario to GAIA. * Pointing autogenbench clone command to the latest branch. * Temporarily disable subsample option. * Updated the GAIA readme to specify how to define a BING API key. * Fixed and re-enabled the subsample option. * Added a draft of a blog post. * Updated authors. * Incorporating Gagan's feedback. * Fixed code formatting. * Updated the help string in the docs. * Light editing of the AutoGenBench blogpost. * Support filtering on model tags. * Added websurfer dependencies to Dockerfile. * Renamed testbed -> autogenbench * Attempting to fix formatting. * Added more gracefull handling of task timeouts (the script is allowed to terminate before Docker is stopped). * Updated the blogpost based on Saleema's and Julia's feedback. * Fixed formatting... again. * Added a main MANIFEST to list available scenarios. * Limit main manifest to directories. * Manifests now use relative paths. * All manifests are now relative. * Updated the contributing guide, and address windows path issues. * Updated the version. Fixed formatting. * Fixed formatting. * De-listing Examples, since it has no clear tabulate criteria. * Updated email in pyproject * typo in blogpost * wording --------- Co-authored-by: Qingyun Wu <qingyun.wu@psu.edu> Co-authored-by: Qingyun Wu <qingyun0327@gmail.com>
27 lines
750 B
Python
27 lines
750 B
Python
import os
|
|
import sys
|
|
import json
|
|
from autogenbench.tabulate_cmd import default_tabulate
|
|
|
|
|
|
def scorer(instance_dir):
|
|
checker_messages = os.path.join(instance_dir, "checker_messages.json")
|
|
if os.path.isfile(checker_messages):
|
|
with open(checker_messages, "rt") as fh:
|
|
messages = json.loads(fh.read())["checker_proxy"]
|
|
results = messages[-1]["content"].lower()
|
|
if "the answer is correct" in results or "the answer is approximated but should be correct" in results:
|
|
return True
|
|
else:
|
|
return False
|
|
else:
|
|
return None
|
|
|
|
|
|
def main(args):
|
|
default_tabulate(args, scorer=scorer)
|
|
|
|
|
|
if __name__ == "__main__" and __package__ is None:
|
|
main(sys.argv)
|