// Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicClientTest.cs using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; using AutoGen.Anthropic.DTO; using AutoGen.Anthropic.Utils; using AutoGen.Tests; using FluentAssertions; using Xunit; namespace AutoGen.Anthropic.Tests; public class AnthropicClientTests { [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientChatCompletionTestAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude3Haiku; request.Stream = false; request.MaxTokens = 100; request.Messages = new List() { new ChatMessage("user", "Hello world") }; ChatCompletionResponse response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); Assert.NotNull(response); Assert.NotNull(response.Content); Assert.NotEmpty(response.Content); response.Content.Count.Should().Be(1); response.Content.First().Should().BeOfType(); var textContent = (TextContent)response.Content.First(); Assert.Equal("text", textContent.Type); Assert.NotNull(response.Usage); response.Usage.OutputTokens.Should().BeGreaterThan(0); } [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientStreamingChatCompletionTestAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude3Haiku; request.Stream = true; request.MaxTokens = 500; request.SystemMessage = [ SystemMessage.CreateSystemMessage( "You are a helpful assistant that convert input to json object, use JSON format.") ]; request.Messages = new List() { new("user", "name: John, age: 41, email: g123456@gmail.com") }; var response = anthropicClient.StreamingChatCompletionsAsync(request, CancellationToken.None); var results = await response.ToListAsync(); results.Count.Should().BeGreaterThan(0); // Merge the chunks. StringBuilder sb = new(); foreach (ChatCompletionResponse result in results) { if (result.Delta is not null && !string.IsNullOrEmpty(result.Delta.Text)) { sb.Append(result.Delta.Text); } } string resultContent = sb.ToString(); Assert.NotNull(resultContent); var person = JsonSerializer.Deserialize(resultContent); Assert.NotNull(person); person.Name.Should().Be("John"); person.Age.Should().Be(41); person.Email.Should().Be("g123456@gmail.com"); Assert.NotNull(results.First().StreamingMessage); results.First().StreamingMessage!.Role.Should().Be("assistant"); } [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientImageChatCompletionTestAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude3Haiku; request.Stream = false; request.MaxTokens = 100; request.SystemMessage = [ SystemMessage.CreateSystemMessage( "You are a LLM that is suppose to describe the content of the image. Give me a description of the provided image."), ]; var base64Image = await AnthropicTestUtils.Base64FromImageAsync("square.png"); var messages = new List { new("user", [ new ImageContent { Source = new ImageSource {MediaType = "image/png", Data = base64Image} } ]) }; request.Messages = messages; var response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); Assert.NotNull(response); Assert.NotNull(response.Content); Assert.NotEmpty(response.Content); response.Content.Count.Should().Be(1); response.Content.First().Should().BeOfType(); var textContent = (TextContent)response.Content.First(); Assert.Equal("text", textContent.Type); Assert.NotNull(response.Usage); response.Usage.OutputTokens.Should().BeGreaterThan(0); } [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientTestToolsAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude3Haiku; request.Stream = false; request.MaxTokens = 100; request.Messages = new List() { new("user", "Use the stock price tool to look for MSFT. Your response should only be the tool.") }; request.Tools = new List() { AnthropicTestUtils.StockTool }; ChatCompletionResponse response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); Assert.NotNull(response.Content); Assert.True(response.Content.First() is ToolUseContent); ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); Assert.Equal("get_stock_price", toolUseContent.Name); Assert.NotNull(toolUseContent.Input); Assert.True(toolUseContent.Input is JsonNode); JsonNode jsonNode = toolUseContent.Input; Assert.Equal("{\"ticker\":\"MSFT\"}", jsonNode.ToJsonString()); } [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientTestToolChoiceAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude3Haiku; request.Stream = false; request.MaxTokens = 100; request.Messages = new List() { new("user", "What is the weather today? Your response should only be the tool.") }; request.Tools = new List() { AnthropicTestUtils.StockTool, AnthropicTestUtils.WeatherTool }; // Force to use get_stock_price even though the prompt is about weather request.ToolChoice = ToolChoice.ToolUse("get_stock_price"); ChatCompletionResponse response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); Assert.NotNull(response.Content); Assert.True(response.Content.First() is ToolUseContent); ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); Assert.Equal("get_stock_price", toolUseContent.Name); Assert.NotNull(toolUseContent.Input); Assert.True(toolUseContent.Input is JsonNode); } [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicClientChatCompletionCacheControlTestAsync() { var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); var request = new ChatCompletionRequest(); request.Model = AnthropicConstants.Claude35Sonnet; request.Stream = false; request.MaxTokens = 100; request.SystemMessage = [ SystemMessage.CreateSystemMessageWithCacheControl( $"You are an LLM that is great at remembering stories {AnthropicTestUtils.LongStory}"), ]; request.Messages = [ new ChatMessage("user", "What should i know about Bob?") ]; var response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); response.Usage.Should().NotBeNull(); // There's no way to clear the cache. Running the assert frequently may cause this to fail because the cache is already been created // response.Usage!.CreationInputTokens.Should().BeGreaterThan(0); // The cache reduces the input tokens. We expect the input tokens to be less the large system prompt and only the user message response.Usage!.InputTokens.Should().BeLessThan(20); request.Messages = [ new ChatMessage("user", "Summarize the story of bob") ]; response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); response.Usage.Should().NotBeNull(); response.Usage!.CacheReadInputTokens.Should().BeGreaterThan(0); response.Usage!.InputTokens.Should().BeLessThan(20); // Should not use the cache request.SystemMessage = [ SystemMessage.CreateSystemMessage("You are a helpful assistant.") ]; request.Messages = [ new ChatMessage("user", "What are some text editors I could use to write C#?") ]; response = await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); response.Usage!.CacheReadInputTokens.Should().Be(0); } private sealed class Person { [JsonPropertyName("name")] public string Name { get; set; } = string.Empty; [JsonPropertyName("age")] public int Age { get; set; } [JsonPropertyName("email")] public string Email { get; set; } = string.Empty; } }