2024-11-22 13:51:08 -08:00
// Copyright (c) Microsoft Corporation. All rights reserved.
// RolePlayToolCallOrchestratorTests.cs
using System ;
using System.Collections.Generic ;
using System.Threading.Tasks ;
using AutoGen.OpenAI.Orchestrator ;
using AutoGen.Tests ;
using Azure.AI.OpenAI ;
using FluentAssertions ;
using Moq ;
using OpenAI ;
using OpenAI.Chat ;
using Xunit ;
namespace AutoGen.OpenAI.Tests ;
2025-02-03 11:49:08 -05:00
[Trait("Category", "UnitV1")]
2024-11-22 13:51:08 -08:00
public class RolePlayToolCallOrchestratorTests
{
[Fact]
public async Task ItReturnNullWhenNoCandidateIsAvailableAsync ( )
{
var chatClient = Mock . Of < ChatClient > ( ) ;
var orchestrator = new RolePlayToolCallOrchestrator ( chatClient ) ;
var context = new OrchestrationContext
{
Candidates = [ ] ,
ChatHistory = [ ] ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker . Should ( ) . BeNull ( ) ;
}
[Fact]
public async Task ItReturnCandidateWhenOnlyOneCandidateIsAvailableAsync ( )
{
var chatClient = Mock . Of < ChatClient > ( ) ;
var alice = new EchoAgent ( "Alice" ) ;
var orchestrator = new RolePlayToolCallOrchestrator ( chatClient ) ;
var context = new OrchestrationContext
{
Candidates = [ alice ] ,
ChatHistory = [ ] ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker . Should ( ) . Be ( alice ) ;
}
[Fact]
public async Task ItSelectNextSpeakerFromWorkflowIfProvided ( )
{
var workflow = new Graph ( ) ;
var alice = new EchoAgent ( "Alice" ) ;
var bob = new EchoAgent ( "Bob" ) ;
var charlie = new EchoAgent ( "Charlie" ) ;
workflow . AddTransition ( Transition . Create ( alice , bob ) ) ;
workflow . AddTransition ( Transition . Create ( bob , charlie ) ) ;
workflow . AddTransition ( Transition . Create ( charlie , alice ) ) ;
var client = Mock . Of < ChatClient > ( ) ;
var orchestrator = new RolePlayToolCallOrchestrator ( client , workflow ) ;
var context = new OrchestrationContext
{
Candidates = [ alice , bob , charlie ] ,
ChatHistory =
[
new TextMessage ( Role . User , "Hello, Bob" , from : "Alice" ) ,
] ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker . Should ( ) . Be ( bob ) ;
}
[Fact]
public async Task ItReturnNullIfNoAvailableAgentFromWorkflowAsync ( )
{
var workflow = new Graph ( ) ;
var alice = new EchoAgent ( "Alice" ) ;
var bob = new EchoAgent ( "Bob" ) ;
workflow . AddTransition ( Transition . Create ( alice , bob ) ) ;
var client = Mock . Of < ChatClient > ( ) ;
var orchestrator = new RolePlayToolCallOrchestrator ( client , workflow ) ;
var context = new OrchestrationContext
{
Candidates = [ alice , bob ] ,
ChatHistory =
[
new TextMessage ( Role . User , "Hello, Alice" , from : "Bob" ) ,
] ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker . Should ( ) . BeNull ( ) ;
}
[ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")]
public async Task GPT_3_5_CoderReviewerRunnerTestAsync ( )
{
var endpoint = Environment . GetEnvironmentVariable ( "AZURE_OPENAI_ENDPOINT" ) ? ? throw new Exception ( "Please set AZURE_OPENAI_ENDPOINT environment variable." ) ;
var key = Environment . GetEnvironmentVariable ( "AZURE_OPENAI_API_KEY" ) ? ? throw new Exception ( "Please set AZURE_OPENAI_API_KEY environment variable." ) ;
var deployName = Environment . GetEnvironmentVariable ( "AZURE_OPENAI_DEPLOY_NAME" ) ? ? throw new Exception ( "Please set AZURE_OPENAI_DEPLOY_NAME environment variable." ) ;
var openaiClient = new AzureOpenAIClient ( new Uri ( endpoint ) , new System . ClientModel . ApiKeyCredential ( key ) ) ;
var chatClient = openaiClient . GetChatClient ( deployName ) ;
await BusinessWorkflowTest ( chatClient ) ;
await CoderReviewerRunnerTestAsync ( chatClient ) ;
}
[ApiKeyFact("OPENAI_API_KEY")]
public async Task GPT_4o_CoderReviewerRunnerTestAsync ( )
{
var apiKey = Environment . GetEnvironmentVariable ( "OPENAI_API_KEY" ) ? ? throw new InvalidOperationException ( "OPENAI_API_KEY is not set" ) ;
var model = "gpt-4o" ;
var openaiClient = new OpenAIClient ( apiKey ) ;
var chatClient = openaiClient . GetChatClient ( model ) ;
await BusinessWorkflowTest ( chatClient ) ;
await CoderReviewerRunnerTestAsync ( chatClient ) ;
}
[ApiKeyFact("OPENAI_API_KEY")]
public async Task GPT_4o_mini_CoderReviewerRunnerTestAsync ( )
{
var apiKey = Environment . GetEnvironmentVariable ( "OPENAI_API_KEY" ) ? ? throw new InvalidOperationException ( "OPENAI_API_KEY is not set" ) ;
var model = "gpt-4o-mini" ;
var openaiClient = new OpenAIClient ( apiKey ) ;
var chatClient = openaiClient . GetChatClient ( model ) ;
await BusinessWorkflowTest ( chatClient ) ;
await CoderReviewerRunnerTestAsync ( chatClient ) ;
}
/// <summary>
/// This test is to mimic the conversation among coder, reviewer and runner.
/// The coder will write the code, the reviewer will review the code, and the runner will run the code.
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
private async Task CoderReviewerRunnerTestAsync ( ChatClient client )
{
var coder = new EchoAgent ( "Coder" ) ;
var reviewer = new EchoAgent ( "Reviewer" ) ;
var runner = new EchoAgent ( "Runner" ) ;
var user = new EchoAgent ( "User" ) ;
var initializeMessage = new List < IMessage >
{
new TextMessage ( Role . User , "Hello, I am user, I will provide the coding task, please write the code first, then review and run it" , from : "User" ) ,
new TextMessage ( Role . User , "Hello, I am coder, I will write the code" , from : "Coder" ) ,
new TextMessage ( Role . User , "Hello, I am reviewer, I will review the code" , from : "Reviewer" ) ,
new TextMessage ( Role . User , "Hello, I am runner, I will run the code" , from : "Runner" ) ,
new TextMessage ( Role . User , "how to print 'hello world' using C#" , from : user . Name ) ,
} ;
var chatHistory = new List < IMessage > ( )
{
new TextMessage ( Role . User , "" "
` ` ` csharp
Console . WriteLine ( "Hello World" ) ;
` ` `
"" ", from: coder.Name),
new TextMessage ( Role . User , "The code looks good" , from : reviewer . Name ) ,
new TextMessage ( Role . User , "The code runs successfully, the output is 'Hello World'" , from : runner . Name ) ,
} ;
var orchestrator = new RolePlayToolCallOrchestrator ( client ) ;
foreach ( var message in chatHistory )
{
var context = new OrchestrationContext
{
Candidates = [ coder , reviewer , runner , user ] ,
ChatHistory = initializeMessage ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker ! . Name . Should ( ) . Be ( message . From ) ;
initializeMessage . Add ( message ) ;
}
// the last next speaker should be the user
var lastSpeaker = await orchestrator . GetNextSpeakerAsync ( new OrchestrationContext
{
Candidates = [ coder , reviewer , runner , user ] ,
ChatHistory = initializeMessage ,
} ) ;
lastSpeaker ! . Name . Should ( ) . Be ( user . Name ) ;
}
// test if the tool call orchestrator still run business workflow when the conversation is not in English
private async Task BusinessWorkflowTest ( ChatClient client )
{
var ceo = new EchoAgent ( "乙方首席执行官" ) ;
var pm = new EchoAgent ( "乙方项目经理" ) ;
var dev = new EchoAgent ( "乙方开发人员" ) ;
var user = new EchoAgent ( "甲方" ) ;
var initializeMessage = new List < IMessage >
{
new TextMessage ( Role . User , "你好,我是你们的甲方" , from : user . Name ) ,
new TextMessage ( Role . User , "你好,我是乙方首席执行官,我将负责对接甲方和给项目经理及开发人员分配任务" , from : ceo . Name ) ,
new TextMessage ( Role . User , "你好,我是乙方项目经理,我将负责项目的进度和质量" , from : pm . Name ) ,
new TextMessage ( Role . User , "你好,我是乙方开发人员 我将负责项目的具体开发" , from : dev . Name ) ,
new TextMessage ( Role . User , "开发一个淘宝, 预算1W" , from : user . Name ) ,
} ;
var workflow = new Graph ( ) ;
workflow . AddTransition ( Transition . Create ( ceo , pm ) ) ;
workflow . AddTransition ( Transition . Create ( ceo , dev ) ) ;
workflow . AddTransition ( Transition . Create ( pm , ceo ) ) ;
workflow . AddTransition ( Transition . Create ( dev , ceo ) ) ;
workflow . AddTransition ( Transition . Create ( user , ceo ) ) ;
workflow . AddTransition ( Transition . Create ( ceo , user ) ) ;
var chatHistory = new List < IMessage > ( )
{
new TextMessage ( Role . User , "" "
项 目 经 理 , 如 何 使 用 1 W预算开发一个淘宝
"" ", from: ceo.Name),
new TextMessage ( Role . User , "" "
对 于 1 万 预 算 开 发 淘 宝 类 网 站 , 以 下 是 关 键 建 议 :
技 术 选 择 :
- 使 用 开 源 电 商 系 统 节 省 成 本 , 选 择 便 宜 但 稳 定 的 云 服 务 器 和 域 名 , 预 算 2000 元 / 年
- 核 心 功 能 优 先
- 人 员 安 排 :
- 找 1 位 全 栈 开 发 , 负 责 系 统 搭 建 ( 6000 元 )
- 1 位 兼 职 UI设计 ( 2000 元 )
- 进 度 规 划 :
- 基 础 功 能 1 个 月 完 成 , 后 续 根 据 运 营 情 况 逐 步 优 化 。
"" ", from: pm.Name),
new TextMessage ( Role . User , "好的,开发人员,请根据项目经理的规划开始开发" , from : ceo . Name ) ,
new TextMessage ( Role . User , "" "
好 的 , 已 开 发 完 毕
` ` ` html
< button class = "taobao-button" onclick = "window.location.href='https://www.taobao.com'" >
Visit Taobao
< / button >
` ` `
"" ", from: dev.Name),
new TextMessage ( Role . User , "好的,项目已完成,甲方请付款" , from : ceo . Name ) ,
} ;
var orchestrator = new RolePlayToolCallOrchestrator ( client , workflow ) ;
foreach ( var message in chatHistory )
{
var context = new OrchestrationContext
{
Candidates = [ ceo , pm , dev , user ] ,
ChatHistory = initializeMessage ,
} ;
var speaker = await orchestrator . GetNextSpeakerAsync ( context ) ;
speaker ! . Name . Should ( ) . Be ( message . From ) ;
initializeMessage . Add ( message ) ;
}
// the last next speaker should be the user
var lastSpeaker = await orchestrator . GetNextSpeakerAsync ( new OrchestrationContext
{
Candidates = [ ceo , pm , dev , user ] ,
ChatHistory = initializeMessage ,
} ) ;
lastSpeaker ! . Name . Should ( ) . Be ( user . Name ) ;
}
}