Made the cost info easier to read (#2356)

* gather_usage_summary has been updated

* updated cost info to 'usage_including_cached_inference' and 'usage_excluding_cached_inference'

* fix: pre-commit formatting for cost_info

* improved cost explanation and doc

* improved cost info doc

* include - exclude

---------

Co-authored-by: Chi Wang <wang.chi@microsoft.com>

autogen/agentchat/chat.py

@@ -25,10 +25,12 @@ class ChatResult:
     """The chat history."""
     summary: str = None
     """A summary obtained from the chat."""
-    cost: tuple = None  # (dict, dict) - (total_cost, actual_cost_with_cache)
-    """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a
-       dictionary of cost information, and total_actual_cost is a dictionary of information on
-       the actual incurred cost with cache."""
+    cost: Dict[str, dict] = None  # keys: "usage_including_cached_inference", "usage_excluding_cached_inference"
+    """The cost of the chat.
+       The value for each usage type is a dictionary containing cost information for that specific type.
+           - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.
+           - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".
+    """
     human_input: List[str] = None
     """A list of human input solicited during the chat."""
 

autogen/agentchat/utils.py

@@ -1,5 +1,5 @@
 import re
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Union
 
 from .agent import Agent
 
@@ -26,33 +26,46 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None:
             ), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm."
 
 
-def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str, any]]:
+def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Dict]]:
     r"""Gather usage summary from all agents.
 
     Args:
         agents: (list): List of agents.
 
     Returns:
-        tuple: (total_usage_summary, actual_usage_summary)
+        dictionary: A dictionary containing two keys:
+          - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.
+          - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".
 
     Example:
 
     ```python
-    total_usage_summary = {
-        "total_cost": 0.0006090000000000001,
-        "gpt-35-turbo": {
-                "cost": 0.0006090000000000001,
-                "prompt_tokens": 242,
-                "completion_tokens": 123,
-                "total_tokens": 365
+    {
+        "usage_including_cached_inference" : {
+            "total_cost": 0.0006090000000000001,
+            "gpt-35-turbo": {
+                    "cost": 0.0006090000000000001,
+                    "prompt_tokens": 242,
+                    "completion_tokens": 123,
+                    "total_tokens": 365
+            },
+        },
+
+        "usage_excluding_cached_inference" : {
+            "total_cost": 0.0006090000000000001,
+            "gpt-35-turbo": {
+                    "cost": 0.0006090000000000001,
+                    "prompt_tokens": 242,
+                    "completion_tokens": 123,
+                    "total_tokens": 365
+            },
         }
     }
     ```
 
     Note:
 
-    `actual_usage_summary` follows the same format.
-    If none of the agents incurred any cost (not having a client), then the total_usage_summary and actual_usage_summary will be `{'total_cost': 0}`.
+    If none of the agents incurred any cost (not having a client), then the usage_including_cached_inference and usage_excluding_cached_inference will be `{'total_cost': 0}`.
     """
 
     def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, Any]) -> None:
@@ -69,15 +82,18 @@ def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str,
                     usage_summary[model]["completion_tokens"] += data.get("completion_tokens", 0)
                     usage_summary[model]["total_tokens"] += data.get("total_tokens", 0)
 
-    total_usage_summary = {"total_cost": 0}
-    actual_usage_summary = {"total_cost": 0}
+    usage_including_cached_inference = {"total_cost": 0}
+    usage_excluding_cached_inference = {"total_cost": 0}
 
     for agent in agents:
         if getattr(agent, "client", None):
-            aggregate_summary(total_usage_summary, agent.client.total_usage_summary)
-            aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary)
+            aggregate_summary(usage_including_cached_inference, agent.client.total_usage_summary)
+            aggregate_summary(usage_excluding_cached_inference, agent.client.actual_usage_summary)
 
-    return total_usage_summary, actual_usage_summary
+    return {
+        "usage_including_cached_inference": usage_including_cached_inference,
+        "usage_excluding_cached_inference": usage_excluding_cached_inference,
+    }
 
 
 def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]:

autogen/code_utils.py

@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)
 
 
 def content_str(content: Union[str, List[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]) -> str:
-    """Converts the `content` field of an OpenAI merssage into a string format.
+    """Converts the `content` field of an OpenAI message into a string format.
 
     This function processes content that may be a string, a list of mixed text and image URLs, or None,
     and converts it into a string. Text is directly appended to the result string, while image URLs are

notebook/agentchat_agentoptimizer.ipynb

@@ -459,7 +459,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.9.13"
   }
  },
  "nbformat": 4,

notebook/agentchat_auto_feedback_from_code_execution.ipynb

@@ -692,6 +692,7 @@
     "        file_content = \"No data found.\"\n",
     "    return \"Analyze the data and write a brief but engaging blog post. \\n Data: \\n\" + file_content\n",
     "\n",
+    "\n",
     "# followup of the previous question\n",
     "chat_res = user_proxy.initiate_chat(\n",
     "    recipient=assistant,\n",

notebook/agentchat_cost_token_tracking.ipynb

@@ -494,8 +494,8 @@
     }
    ],
    "source": [
-    "total_usage_summary, actual_usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",
-    "total_usage_summary"
+    "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",
+    "usage_summary[\"usage_including_cached_inference\"]"
    ]
   }
  ],

notebook/agentchat_groupchat_stateflow.ipynb

@@ -112,7 +112,6 @@
     ")\n",
     "\n",
     "\n",
-    "\n",
     "coder = autogen.AssistantAgent(\n",
     "    name=\"Retrieve_Action_1\",\n",
     "    llm_config=gpt4_config,\n",

notebook/agentchat_image_generation_capability.ipynb

@@ -135,6 +135,7 @@
     "                return content[\"text\"].rstrip().endswith(\"TERMINATE\")\n",
     "    return False\n",
     "\n",
+    "\n",
     "def critic_agent() -> autogen.ConversableAgent:\n",
     "    return autogen.ConversableAgent(\n",
     "        name=\"critic\",\n",

test/agentchat/test_agent_usage.py

@@ -1,21 +1,18 @@
 #!/usr/bin/env python3 -m pytest
 
 import io
+import os
+import sys
 from contextlib import redirect_stdout
 
 import pytest
-from conftest import skip_openai
 from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
 
 import autogen
 from autogen import AssistantAgent, UserProxyAgent, gather_usage_summary
 
-try:
-    import openai
-except ImportError:
-    skip = True
-else:
-    skip = False or skip_openai
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from conftest import skip_openai as skip  # noqa: E402
 
 
 @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
@@ -62,11 +59,11 @@ def test_gathering():
         "gpt-4": {"cost": 0.3, "prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300},
     }
 
-    total_usage, _ = gather_usage_summary([assistant1, assistant2, assistant3])
+    total_usage = gather_usage_summary([assistant1, assistant2, assistant3])
 
-    assert round(total_usage["total_cost"], 8) == 0.6
-    assert round(total_usage["gpt-35-turbo"]["cost"], 8) == 0.3
-    assert round(total_usage["gpt-4"]["cost"], 8) == 0.3
+    assert round(total_usage["usage_including_cached_inference"]["total_cost"], 8) == 0.6
+    assert round(total_usage["usage_including_cached_inference"]["gpt-35-turbo"]["cost"], 8) == 0.3
+    assert round(total_usage["usage_including_cached_inference"]["gpt-4"]["cost"], 8) == 0.3
 
     # test when agent doesn't have client
     user_proxy = UserProxyAgent(
@@ -77,7 +74,10 @@ def test_gathering():
         default_auto_reply="That's all. Thank you.",
     )
 
-    total_usage, acutal_usage = gather_usage_summary([user_proxy])
+    total_usage = gather_usage_summary([user_proxy])
+    total_usage_summary = total_usage["usage_including_cached_inference"]
+
+    print("Total usage summary:", total_usage_summary)
 
 
 @pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")