mirror of
https://github.com/microsoft/autogen.git
synced 2025-11-03 19:29:52 +00:00
378 lines
15 KiB
C#
378 lines
15 KiB
C#
// Copyright (c) Microsoft Corporation. All rights reserved.
|
|
// RolePlayOrchestratorTests.cs
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Net.Http;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using AutoGen.Anthropic;
|
|
using AutoGen.Anthropic.Extensions;
|
|
using AutoGen.Anthropic.Utils;
|
|
using AutoGen.AzureAIInference;
|
|
using AutoGen.AzureAIInference.Extension;
|
|
using AutoGen.Gemini;
|
|
using AutoGen.Mistral;
|
|
using AutoGen.Mistral.Extension;
|
|
using AutoGen.OpenAI;
|
|
using AutoGen.OpenAI.Extension;
|
|
using Azure.AI.Inference;
|
|
using Azure.AI.OpenAI;
|
|
using FluentAssertions;
|
|
using Moq;
|
|
using OpenAI;
|
|
using Xunit;
|
|
|
|
namespace AutoGen.Tests;
|
|
|
|
public class RolePlayOrchestratorTests
|
|
{
|
|
[Fact]
|
|
public async Task ItReturnNextSpeakerTestAsync()
|
|
{
|
|
var admin = Mock.Of<IAgent>();
|
|
Mock.Get(admin).Setup(x => x.Name).Returns("Admin");
|
|
Mock.Get(admin).Setup(x => x.GenerateReplyAsync(
|
|
It.IsAny<IEnumerable<IMessage>>(),
|
|
It.IsAny<GenerateReplyOptions>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.Callback<IEnumerable<IMessage>, GenerateReplyOptions, CancellationToken>((messages, option, _) =>
|
|
{
|
|
// verify prompt
|
|
var rolePlayPrompt = messages.First().GetContent();
|
|
rolePlayPrompt.Should().Contain("You are in a role play game. Carefully read the conversation history and carry on the conversation");
|
|
rolePlayPrompt.Should().Contain("The available roles are:");
|
|
rolePlayPrompt.Should().Contain("Alice,Bob");
|
|
rolePlayPrompt.Should().Contain("From Alice:");
|
|
option.StopSequence.Should().BeEquivalentTo([":"]);
|
|
option.Temperature.Should().Be(0);
|
|
option.MaxToken.Should().Be(128);
|
|
option.Functions.Should().BeNull();
|
|
})
|
|
.ReturnsAsync(new TextMessage(Role.Assistant, "From Alice"));
|
|
|
|
var alice = new EchoAgent("Alice");
|
|
var bob = new EchoAgent("Bob");
|
|
|
|
var orchestrator = new RolePlayOrchestrator(admin);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice, bob],
|
|
ChatHistory = [],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().Be(alice);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItReturnNullWhenNoCandidateIsAvailableAsync()
|
|
{
|
|
var admin = Mock.Of<IAgent>();
|
|
var orchestrator = new RolePlayOrchestrator(admin);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [],
|
|
ChatHistory = [],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().BeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItReturnCandidateWhenOnlyOneCandidateIsAvailableAsync()
|
|
{
|
|
var admin = Mock.Of<IAgent>();
|
|
var alice = new EchoAgent("Alice");
|
|
var orchestrator = new RolePlayOrchestrator(admin);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice],
|
|
ChatHistory = [],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().Be(alice);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItThrowExceptionWhenAdminFailsToFollowPromptAsync()
|
|
{
|
|
var admin = Mock.Of<IAgent>();
|
|
Mock.Get(admin).Setup(x => x.Name).Returns("Admin");
|
|
Mock.Get(admin).Setup(x => x.GenerateReplyAsync(
|
|
It.IsAny<IEnumerable<IMessage>>(),
|
|
It.IsAny<GenerateReplyOptions>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync(new TextMessage(Role.Assistant, "I don't know")); // admin fails to follow the prompt and returns an invalid message
|
|
|
|
var alice = new EchoAgent("Alice");
|
|
var bob = new EchoAgent("Bob");
|
|
|
|
var orchestrator = new RolePlayOrchestrator(admin);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice, bob],
|
|
ChatHistory = [],
|
|
};
|
|
|
|
var action = async () => await orchestrator.GetNextSpeakerAsync(context);
|
|
|
|
await action.Should().ThrowAsync<Exception>()
|
|
.WithMessage("The response from admin is 't know, which is either not in the candidates list or not in the correct format.");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItSelectNextSpeakerFromWorkflowIfProvided()
|
|
{
|
|
var workflow = new Graph();
|
|
var alice = new EchoAgent("Alice");
|
|
var bob = new EchoAgent("Bob");
|
|
var charlie = new EchoAgent("Charlie");
|
|
workflow.AddTransition(Transition.Create(alice, bob));
|
|
workflow.AddTransition(Transition.Create(bob, charlie));
|
|
workflow.AddTransition(Transition.Create(charlie, alice));
|
|
|
|
var admin = Mock.Of<IAgent>();
|
|
var orchestrator = new RolePlayOrchestrator(admin, workflow);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice, bob, charlie],
|
|
ChatHistory =
|
|
[
|
|
new TextMessage(Role.User, "Hello, Bob", from: "Alice"),
|
|
],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().Be(bob);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItReturnNullIfNoAvailableAgentFromWorkflowAsync()
|
|
{
|
|
var workflow = new Graph();
|
|
var alice = new EchoAgent("Alice");
|
|
var bob = new EchoAgent("Bob");
|
|
workflow.AddTransition(Transition.Create(alice, bob));
|
|
|
|
var admin = Mock.Of<IAgent>();
|
|
var orchestrator = new RolePlayOrchestrator(admin, workflow);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice, bob],
|
|
ChatHistory =
|
|
[
|
|
new TextMessage(Role.User, "Hello, Alice", from: "Bob"),
|
|
],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().BeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ItUseCandidatesFromWorflowAsync()
|
|
{
|
|
var workflow = new Graph();
|
|
var alice = new EchoAgent("Alice");
|
|
var bob = new EchoAgent("Bob");
|
|
var charlie = new EchoAgent("Charlie");
|
|
workflow.AddTransition(Transition.Create(alice, bob));
|
|
workflow.AddTransition(Transition.Create(alice, charlie));
|
|
|
|
var admin = Mock.Of<IAgent>();
|
|
Mock.Get(admin).Setup(x => x.GenerateReplyAsync(
|
|
It.IsAny<IEnumerable<IMessage>>(),
|
|
It.IsAny<GenerateReplyOptions>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.Callback<IEnumerable<IMessage>, GenerateReplyOptions, CancellationToken>((messages, option, _) =>
|
|
{
|
|
messages.First().IsSystemMessage().Should().BeTrue();
|
|
|
|
// verify prompt
|
|
var rolePlayPrompt = messages.First().GetContent();
|
|
rolePlayPrompt.Should().Contain("Bob,Charlie");
|
|
rolePlayPrompt.Should().Contain("From Bob:");
|
|
option.StopSequence.Should().BeEquivalentTo([":"]);
|
|
option.Temperature.Should().Be(0);
|
|
option.MaxToken.Should().Be(128);
|
|
option.Functions.Should().BeEmpty();
|
|
})
|
|
.ReturnsAsync(new TextMessage(Role.Assistant, "From Bob"));
|
|
var orchestrator = new RolePlayOrchestrator(admin, workflow);
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [alice, bob],
|
|
ChatHistory =
|
|
[
|
|
new TextMessage(Role.User, "Hello, Bob", from: "Alice"),
|
|
],
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker.Should().Be(bob);
|
|
}
|
|
|
|
[ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")]
|
|
public async Task GPT_3_5_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new Exception("Please set AZURE_OPENAI_ENDPOINT environment variable.");
|
|
var key = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY") ?? throw new Exception("Please set AZURE_OPENAI_API_KEY environment variable.");
|
|
var deployName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOY_NAME") ?? throw new Exception("Please set AZURE_OPENAI_DEPLOY_NAME environment variable.");
|
|
var openaiClient = new AzureOpenAIClient(new Uri(endpoint), new System.ClientModel.ApiKeyCredential(key));
|
|
var openAIChatAgent = new OpenAIChatAgent(
|
|
chatClient: openaiClient.GetChatClient(deployName),
|
|
name: "assistant")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(openAIChatAgent);
|
|
}
|
|
|
|
[ApiKeyFact("OPENAI_API_KEY")]
|
|
public async Task GPT_4o_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set");
|
|
var model = "gpt-4o";
|
|
var openaiClient = new OpenAIClient(apiKey);
|
|
var openAIChatAgent = new OpenAIChatAgent(
|
|
chatClient: openaiClient.GetChatClient(model),
|
|
name: "assistant")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(openAIChatAgent);
|
|
}
|
|
|
|
[ApiKeyFact("OPENAI_API_KEY")]
|
|
public async Task GPT_4o_mini_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set");
|
|
var model = "gpt-4o-mini";
|
|
var openaiClient = new OpenAIClient(apiKey);
|
|
var openAIChatAgent = new OpenAIChatAgent(
|
|
chatClient: openaiClient.GetChatClient(model),
|
|
name: "assistant")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(openAIChatAgent);
|
|
}
|
|
|
|
[ApiKeyFact("GOOGLE_GEMINI_API_KEY")]
|
|
public async Task GoogleGemini_1_5_flash_001_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("GOOGLE_GEMINI_API_KEY") ?? throw new InvalidOperationException("GOOGLE_GEMINI_API_KEY is not set");
|
|
var geminiAgent = new GeminiChatAgent(
|
|
name: "gemini",
|
|
model: "gemini-1.5-flash-001",
|
|
apiKey: apiKey)
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(geminiAgent);
|
|
}
|
|
|
|
[ApiKeyFact("ANTHROPIC_API_KEY")]
|
|
public async Task Claude3_Haiku_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? throw new Exception("Please set ANTHROPIC_API_KEY environment variable.");
|
|
var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey);
|
|
|
|
var agent = new AnthropicClientAgent(
|
|
client,
|
|
name: "AnthropicAgent",
|
|
AnthropicConstants.Claude3Haiku,
|
|
systemMessage: "You are a helpful AI assistant that convert user message to upper case")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(agent);
|
|
}
|
|
|
|
[ApiKeyFact("MISTRAL_API_KEY")]
|
|
public async Task Mistra_7b_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("MISTRAL_API_KEY") ?? throw new InvalidOperationException("MISTRAL_API_KEY is not set.");
|
|
var client = new MistralClient(apiKey: apiKey);
|
|
|
|
var agent = new MistralClientAgent(
|
|
client: client,
|
|
name: "MistralClientAgent",
|
|
model: "open-mistral-7b")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(agent);
|
|
}
|
|
|
|
[ApiKeyFact("GH_API_KEY")]
|
|
public async Task LLaMA_3_1_CoderReviewerRunnerTestAsync()
|
|
{
|
|
var apiKey = Environment.GetEnvironmentVariable("GH_API_KEY") ?? throw new InvalidOperationException("GH_API_KEY is not set.");
|
|
var endPoint = "https://models.inference.ai.azure.com";
|
|
|
|
var chatCompletionClient = new ChatCompletionsClient(new Uri(endPoint), new Azure.AzureKeyCredential(apiKey));
|
|
var agent = new ChatCompletionsClientAgent(
|
|
chatCompletionsClient: chatCompletionClient,
|
|
name: "assistant",
|
|
modelName: "Meta-Llama-3.1-70B-Instruct")
|
|
.RegisterMessageConnector();
|
|
|
|
await CoderReviewerRunnerTestAsync(agent);
|
|
}
|
|
|
|
/// <summary>
|
|
/// This test is to mimic the conversation among coder, reviewer and runner.
|
|
/// The coder will write the code, the reviewer will review the code, and the runner will run the code.
|
|
/// </summary>
|
|
/// <param name="admin"></param>
|
|
/// <returns></returns>
|
|
public async Task CoderReviewerRunnerTestAsync(IAgent admin)
|
|
{
|
|
var coder = new EchoAgent("Coder");
|
|
var reviewer = new EchoAgent("Reviewer");
|
|
var runner = new EchoAgent("Runner");
|
|
var user = new EchoAgent("User");
|
|
var initializeMessage = new List<IMessage>
|
|
{
|
|
new TextMessage(Role.User, "Hello, I am user, I will provide the coding task, please write the code first, then review and run it", from: "User"),
|
|
new TextMessage(Role.User, "Hello, I am coder, I will write the code", from: "Coder"),
|
|
new TextMessage(Role.User, "Hello, I am reviewer, I will review the code", from: "Reviewer"),
|
|
new TextMessage(Role.User, "Hello, I am runner, I will run the code", from: "Runner"),
|
|
new TextMessage(Role.User, "how to print 'hello world' using C#", from: user.Name),
|
|
};
|
|
|
|
var chatHistory = new List<IMessage>()
|
|
{
|
|
new TextMessage(Role.User, """
|
|
```csharp
|
|
Console.WriteLine("Hello World");
|
|
```
|
|
""", from: coder.Name),
|
|
new TextMessage(Role.User, "The code looks good", from: reviewer.Name),
|
|
new TextMessage(Role.User, "The code runs successfully, the output is 'Hello World'", from: runner.Name),
|
|
};
|
|
|
|
var orchestrator = new RolePlayOrchestrator(admin);
|
|
foreach (var message in chatHistory)
|
|
{
|
|
var context = new OrchestrationContext
|
|
{
|
|
Candidates = [coder, reviewer, runner, user],
|
|
ChatHistory = initializeMessage,
|
|
};
|
|
|
|
var speaker = await orchestrator.GetNextSpeakerAsync(context);
|
|
speaker!.Name.Should().Be(message.From);
|
|
initializeMessage.Add(message);
|
|
}
|
|
|
|
// the last next speaker should be the user
|
|
var lastSpeaker = await orchestrator.GetNextSpeakerAsync(new OrchestrationContext
|
|
{
|
|
Candidates = [coder, reviewer, runner, user],
|
|
ChatHistory = initializeMessage,
|
|
});
|
|
|
|
lastSpeaker!.Name.Should().Be(user.Name);
|
|
}
|
|
}
|