yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-11-01 10:19:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Knowledge abstraction to AiAgent (#25)

Browse Source 
* make ghClient fetch synchronous

* refactor memory, extract knowledge adding to AiAgent
This commit is contained in:
Kosta Petan 2024-03-28 13:34:33 +01:00 committed by GitHub
parent 3d90292aa8
commit fda381a82a
14 changed files with 147 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.editorconfig Normal file
View File

@ -0,0 +1,23 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true
# C# files
[*.cs]
#### Core EditorConfig Options ####
# Indentation and spacing
indent_size = 4
indent_style = space
tab_width = 4
#### .NET Coding Conventions ####
# this. and Me. preferences
dotnet_style_qualification_for_method = true
#### Diagnostic configuration ####
dotnet_diagnostic.SKEXP0001.severity = none
dotnet_diagnostic.SKEXP0020.severity = none
dotnet_diagnostic.SKEXP0010.severity = none

32
src/libs/Microsoft.AI.Agents/Abstractions/AiAgent.cs
View File

@ -1,5 +1,7 @@
using System.Text;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime;
namespace Microsoft.AI.Agents.Abstractions;
@ -7,11 +9,15 @@ namespace Microsoft.AI.Agents.Abstractions;
public abstract class AiAgent<T> : Agent
{
public AiAgent(
[PersistentState("state", "messages")] IPersistentState<AgentState<T>> state)
[PersistentState("state", "messages")] IPersistentState<AgentState<T>> state, ISemanticTextMemory memory, Kernel kernel)
{
_state = state;
_memory = memory;
_kernel = kernel;
}
protected IPersistentState<AgentState<T>> _state;
private readonly ISemanticTextMemory _memory;
private readonly Kernel _kernel;
protected void AddToHistory(string message, ChatUserType userType)
{
@ -30,19 +36,33 @@ public abstract class AiAgent<T> : Agent
return string.Join("\n", _state.State.History.Select(message => $"{message.UserType}: {message.Message}"));
}
protected virtual async Task<string> CallFunction(string template, KernelArguments arguments, Kernel kernel, OpenAIPromptExecutionSettings? settings = null)
protected virtual async Task<string> CallFunction(string template, KernelArguments arguments, OpenAIPromptExecutionSettings? settings = null)
{
var propmptSettings = (settings == null) ? new OpenAIPromptExecutionSettings { MaxTokens = 18000, Temperature = 0.8, TopP = 1 }
: settings;
var function = kernel.CreateFunctionFromPrompt(template, propmptSettings);
var result = (await kernel.InvokeAsync(function, arguments)).ToString();
var function = _kernel.CreateFunctionFromPrompt(template, propmptSettings);
var result = (await _kernel.InvokeAsync(function, arguments)).ToString();
AddToHistory(result, ChatUserType.Agent);
return result;
}
protected async Task<T> ShareContext()
/// <summary>
/// Adds knowledge to the
/// </summary>
/// <param name="instruction">The instruction string that uses the value of !index! as a placeholder to inject the data. Example:"Consider the following architectural guidelines: {waf}" </param>
/// <param name="index">Knowledge index</param>
/// <param name="arguments">The sk arguments, "input" is the argument </param>
/// <returns></returns>
protected async Task<KernelArguments> AddKnowledge(string instruction, string index, KernelArguments arguments)
{
return _state.State.Data;
var documents = _memory.SearchAsync(index, arguments["input"].ToString(), 5);
var kbStringBuilder = new StringBuilder();
await foreach (var doc in documents)
{
kbStringBuilder.AppendLine($"{doc.Metadata.Text}");
}
arguments[index] = instruction.Replace($"!{index}!", $"{kbStringBuilder}");
return arguments;
}
}

1
src/libs/Microsoft.AI.Agents/Microsoft.AI.Agents.csproj
View File

@ -8,7 +8,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" />
<PackageReference Include="Microsoft.Orleans.Runtime" Version="8.0.0" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.Orleans.Streaming" Version="8.0.0" />
</ItemGroup>

10
src/sample-apps/gh-flow/Agents/Architect/Architect.cs
View File

@ -1,5 +1,6 @@
using Microsoft.AI.Agents.Abstractions;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime;
using Orleans.Streams;
@ -8,17 +9,16 @@ namespace Microsoft.AI.DevTeam;
// The architect has Org+Repo scope and is holding the knowledge of the high level architecture of the project
[ImplicitStreamSubscription(Consts.MainNamespace)]
public class Architect : AzureAiAgent<ArchitectState>
public class Architect : AiAgent<ArchitectState>
{
protected override string Namespace => Consts.MainNamespace;
public Architect([PersistentState("state", "messages")] IPersistentState<AgentState<ArchitectState>> state, IKernelMemory memory)
: base(state, memory)
public Architect([PersistentState("state", "messages")] IPersistentState<AgentState<ArchitectState>> state, ISemanticTextMemory memory, Kernel kernel)
: base(state, memory, kernel)
{
}
public override Task HandleEvent(Event item, StreamSequenceToken? token)
{
// throw new NotImplementedException();
return Task.CompletedTask;
}
}

30
src/sample-apps/gh-flow/Agents/AzureAiAgent.cs
View File

@ -1,30 +0,0 @@
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Orleans.Runtime;
using Microsoft.AI.Agents.Abstractions;
using Microsoft.SemanticKernel.Connectors.OpenAI;
namespace Microsoft.AI.DevTeam;
public abstract class AzureAiAgent<T> : AiAgent<T>
{
private readonly IKernelMemory _memory;
public AzureAiAgent([PersistentState("state", "messages")] IPersistentState<AgentState<T>> state, IKernelMemory memory) : base(state)
{
_memory = memory;
}
protected async Task<KernelArguments> AddWafContext(IKernelMemory memory, KernelArguments arguments)
{
var waf = await memory.AskAsync(arguments["input"].ToString(), index: "waf");
if (!waf.NoResult) arguments["wafContext"] = $"Consider the following architectural guidelines: ${waf.Result}";
return arguments;
}
protected override async Task<string> CallFunction(string template, KernelArguments arguments, Kernel kernel, OpenAIPromptExecutionSettings? settings = null)
{
var wafArguments = await AddWafContext(_memory, arguments);
return await base.CallFunction(template, wafArguments, kernel, settings);
}
}

49
src/sample-apps/gh-flow/Agents/Developer/Developer.cs
View File

@ -1,23 +1,22 @@
using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime;
using Orleans.Streams;
namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)]
public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps
public class Dev : AiAgent<DeveloperState>, IDevelopApps
{
protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<Dev> _logger;
public Dev([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperState>> state, Kernel kernel, IKernelMemory memory, ILogger<Dev> logger)
: base(state, memory)
public Dev([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<Dev> logger)
: base(state, memory, kernel)
{
_kernel = kernel;
_logger = logger;
}
@ -65,7 +64,9 @@ public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps
{
// TODO: ask the architect for the high level architecture as well as the files structure of the project
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)};
return await CallFunction(DeveloperSkills.Implement, context, _kernel);
var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf",context);
return await CallFunction(DeveloperSkills.Implement, enhancedContext);
}
catch (Exception ex)
{
@ -73,40 +74,6 @@ public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps
return default;
}
}
// public async Task<UnderstandingResult> BuildUnderstanding(string content)
// {
// try
// {
// var explainFunction = _kernel.CreateSemanticFunction(Developer.Explain, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
// var consolidateFunction = _kernel.CreateSemanticFunction(Developer.ConsolidateUnderstanding, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
// var explainContext = new ContextVariables();
// explainContext.Set("input", content);
// var explainResult = await _kernel.RunAsync(explainContext, explainFunction);
// var explainMesage = explainResult.ToString();
// var consolidateContext = new ContextVariables();
// consolidateContext.Set("input", _state.State.Understanding);
// consolidateContext.Set("newUnderstanding", explainMesage);
// var consolidateResult = await _kernel.RunAsync(consolidateContext, consolidateFunction);
// var consolidateMessage = consolidateResult.ToString();
// _state.State.Understanding = consolidateMessage;
// await _state.WriteStateAsync();
// return new UnderstandingResult
// {
// NewUnderstanding = consolidateMessage,
// Explanation = explainMesage
// };
// }
// catch (Exception ex)
// {
// _logger.LogError(ex, "Error building understanding");
// return default;
// }
// }
}
[GenerateSerializer]

19
src/sample-apps/gh-flow/Agents/DeveloperLead/DeveloperLead.cs
View File

@ -1,23 +1,20 @@
using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime;
using Orleans.Streams;
namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)]
public class DeveloperLead : AzureAiAgent<DeveloperLeadState>, ILeadDevelopers
public class DeveloperLead : AiAgent<DeveloperLeadState>, ILeadDevelopers
{
protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<DeveloperLead> _logger;
public DeveloperLead([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperLeadState>> state, Kernel kernel, IKernelMemory memory, ILogger<DeveloperLead> logger)
: base(state, memory)
public DeveloperLead([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperLeadState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<DeveloperLead> logger)
: base(state, memory, kernel)
{
_kernel = kernel;
_logger = logger;
}
@ -64,8 +61,10 @@ public class DeveloperLead : AzureAiAgent<DeveloperLeadState>, ILeadDevelopers
{
// TODO: Ask the architect for the existing high level architecture
// as well as the file structure
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)};
return await CallFunction(DevLeadSkills.Plan, context, _kernel);
var context = new KernelArguments { ["input"] = AppendChatHistory(ask) };
var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf", context);
return await CallFunction(DevLeadSkills.Plan, enhancedContext);
}
catch (Exception ex)
{
@ -77,7 +76,7 @@ public class DeveloperLead : AzureAiAgent<DeveloperLeadState>, ILeadDevelopers
public interface ILeadDevelopers
{
public Task<string> CreatePlan(string ask);
public Task<string> CreatePlan(string ask);
}
[GenerateSerializer]

15
src/sample-apps/gh-flow/Agents/ProductManager/ProductManager.cs
View File

@ -1,24 +1,21 @@
using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime;
using Orleans.Streams;
namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)]
public class ProductManager : AzureAiAgent<ProductManagerState>, IManageProducts
public class ProductManager : AiAgent<ProductManagerState>, IManageProducts
{
protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<ProductManager> _logger;
public ProductManager([PersistentState("state", "messages")] IPersistentState<AgentState<ProductManagerState>> state, Kernel kernel, IKernelMemory memory, ILogger<ProductManager> logger)
: base(state, memory)
public ProductManager([PersistentState("state", "messages")] IPersistentState<AgentState<ProductManagerState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<ProductManager> logger)
: base(state, memory, kernel)
{
_kernel = kernel;
//_memory = memory;
_logger = logger;
}
@ -63,7 +60,9 @@ public class ProductManager : AzureAiAgent<ProductManagerState>, IManageProducts
try
{
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)};
return await CallFunction(PMSkills.Readme, context, _kernel);
var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf",context);
return await CallFunction(PMSkills.Readme, enhancedContext);
}
catch (Exception ex)
{

45
src/sample-apps/gh-flow/Program.cs
View File

@ -9,12 +9,14 @@ using Octokit.Webhooks.AspNetCore;
using Azure.Identity;
using Microsoft.Extensions.Azure;
using Microsoft.Extensions.Http.Resilience;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Connectors.OpenAI;
var builder = WebApplication.CreateBuilder(args);
builder.Services.AddSingleton<WebhookEventProcessor, GithubWebHookProcessor>();
builder.Services.AddTransient(CreateKernel);
builder.Services.AddSingleton<IKernelMemory>(CreateMemory);
builder.Services.AddTransient(CreateMemory);
builder.Services.AddHttpClient();
builder.Services.AddSingleton(s =>
@ -22,7 +24,7 @@ builder.Services.AddSingleton(s =>
var ghOptions = s.GetService<IOptions<GithubOptions>>();
var logger = s.GetService<ILogger<GithubAuthService>>();
var ghService = new GithubAuthService(ghOptions, logger);
var client = ghService.GetGitHubClient().Result;
var client = ghService.GetGitHubClient();
return client;
});
@ -96,29 +98,24 @@ app.Map("/dashboard", x => x.UseOrleansDashboard());
app.Run();
static IKernelMemory CreateMemory(IServiceProvider provider)
static ISemanticTextMemory CreateMemory(IServiceProvider provider)
{
var qdrantConfig = provider.GetService<IOptions<QdrantOptions>>().Value;
var openAiConfig = provider.GetService<IOptions<OpenAIOptions>>().Value;
return new KernelMemoryBuilder()
.WithQdrantMemoryDb(qdrantConfig.Endpoint)
.WithAzureOpenAITextGeneration(new AzureOpenAIConfig
{
APIType = AzureOpenAIConfig.APITypes.ChatCompletion,
Endpoint = openAiConfig.Endpoint,
Deployment = openAiConfig.DeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = openAiConfig.ApiKey
})
.WithAzureOpenAITextEmbeddingGeneration(new AzureOpenAIConfig
{
APIType = AzureOpenAIConfig.APITypes.EmbeddingGeneration,
Endpoint = openAiConfig.Endpoint,
Deployment =openAiConfig.EmbeddingDeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = openAiConfig.ApiKey
})
.Build<MemoryServerless>();
var qdrantConfig = provider.GetService<IOptions<QdrantOptions>>().Value;
var loggerFactory = LoggerFactory.Create(builder =>
{
builder
.SetMinimumLevel(LogLevel.Debug)
.AddConsole()
.AddDebug();
});
var memoryBuilder = new MemoryBuilder();
return memoryBuilder.WithLoggerFactory(loggerFactory)
.WithQdrantMemoryStore(qdrantConfig.Endpoint, qdrantConfig.VectorSize)
.WithAzureOpenAITextEmbeddingGeneration(openAiConfig.EmbeddingDeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey)
.Build();
}
static Kernel CreateKernel(IServiceProvider provider)

4
src/sample-apps/gh-flow/Services/GithubAuthService.cs
View File

@ -44,7 +44,7 @@ public class GithubAuthService
return new JwtSecurityTokenHandler().WriteToken(token);
}
public async Task<GitHubClient> GetGitHubClient()
public GitHubClient GetGitHubClient()
{
try
{
@ -53,7 +53,7 @@ public class GithubAuthService
{
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer)
};
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId);
var response = appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId).Result;
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{
Credentials = new Credentials(response.Token)

3
src/sample-apps/gh-flow/gh-flow.csproj
View File

@ -21,7 +21,6 @@
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240318.1" />
<PackageReference Include="Microsoft.Orleans.Server" Version="8.0.0" />
<PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" />
@ -45,7 +44,7 @@
<PackageReference Include="Azure.Identity" Version="1.11.0-beta.1" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.4.1" />
<PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240318.1" />
</ItemGroup>

63
src/sample-apps/seed-memory/Program.cs
View File

@ -1,12 +1,17 @@
using Microsoft.Extensions.Logging;
using Microsoft.KernelMemory;
using System.Reflection;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Memory;
using UglyToad.PdfPig;
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
class Program
{
static string WafFileName = "azure-well-architected.pdf";
static async Task Main(string[] args)
{
var kernelSettings = KernelSettings.LoadSettings();
var kernelSettings = KernelSettings.LoadSettings();
using ILoggerFactory loggerFactory = LoggerFactory.Create(builder =>
{
@ -15,34 +20,38 @@ class Program
.AddConsole()
.AddDebug();
});
var memoryBuilder = new MemoryBuilder();
var memory = memoryBuilder.WithLoggerFactory(loggerFactory)
.WithQdrantMemoryStore(kernelSettings.QdrantEndpoint, 1536)
.WithAzureOpenAITextEmbeddingGeneration(kernelSettings.EmbeddingDeploymentOrModelId,kernelSettings.Endpoint, kernelSettings.ApiKey)
.Build();
var memory = new KernelMemoryBuilder()
.WithQdrantMemoryDb(kernelSettings.QdrantEndpoint)
.WithAzureOpenAITextGeneration(new AzureOpenAIConfig
{
APIType = AzureOpenAIConfig.APITypes.ChatCompletion,
Endpoint =kernelSettings.Endpoint,
Deployment = kernelSettings.DeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = kernelSettings.ApiKey
})
.WithAzureOpenAITextEmbeddingGeneration(new AzureOpenAIConfig
{
APIType = AzureOpenAIConfig.APITypes.EmbeddingGeneration,
Endpoint = kernelSettings.Endpoint,
Deployment =kernelSettings.EmbeddingDeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = kernelSettings.ApiKey
})
.Build<MemoryServerless>();
await ImportDocumentAsync(memory, WafFileName);
}
public static async Task ImportDocumentAsync(IKernelMemory memory, string filename)
public static async Task ImportDocumentAsync(ISemanticTextMemory memory, string filename)
{
await memory.ImportDocumentAsync(new Document("wafdoc")
.AddFiles([
filename
]), index: "waf");
var currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
var filePath = Path.Combine(currentDirectory, filename);
using var pdfDocument = PdfDocument.Open(File.OpenRead(filePath));
var pages = pdfDocument.GetPages();
foreach (var page in pages)
{
try
{
var text = ContentOrderTextExtractor.GetText(page);
var descr = text.Take(100);
await memory.SaveInformationAsync(
collection: "waf",
text: text,
id: $"{Guid.NewGuid()}",
description: $"Document: {descr}");
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
}

2
src/sample-apps/seed-memory/config/KernelSettings.cs
View File

@ -72,7 +72,7 @@ internal class KernelSettings
}
var configuration = new ConfigurationBuilder()
.SetBasePath(System.IO.Directory.GetCurrentDirectory())
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile(configFile, optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();

10
src/sample-apps/seed-memory/seed-memory.csproj
View File

@ -11,8 +11,14 @@
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240318.1" />
<PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240318.1" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20240324-e7896" />
</ItemGroup>
<ItemGroup>