Knowledge abstraction to AiAgent (#25)

* make ghClient fetch synchronous

* refactor memory, extract knowledge adding to AiAgent
This commit is contained in:
Kosta Petan 2024-03-28 13:34:33 +01:00 committed by GitHub
parent 3d90292aa8
commit fda381a82a
14 changed files with 147 additions and 159 deletions

23
.editorconfig Normal file
View File

@ -0,0 +1,23 @@
# Remove the line below if you want to inherit .editorconfig settings from higher directories
root = true
# C# files
[*.cs]
#### Core EditorConfig Options ####
# Indentation and spacing
indent_size = 4
indent_style = space
tab_width = 4
#### .NET Coding Conventions ####
# this. and Me. preferences
dotnet_style_qualification_for_method = true
#### Diagnostic configuration ####
dotnet_diagnostic.SKEXP0001.severity = none
dotnet_diagnostic.SKEXP0020.severity = none
dotnet_diagnostic.SKEXP0010.severity = none

View File

@ -1,5 +1,7 @@
using System.Text;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime; using Orleans.Runtime;
namespace Microsoft.AI.Agents.Abstractions; namespace Microsoft.AI.Agents.Abstractions;
@ -7,11 +9,15 @@ namespace Microsoft.AI.Agents.Abstractions;
public abstract class AiAgent<T> : Agent public abstract class AiAgent<T> : Agent
{ {
public AiAgent( public AiAgent(
[PersistentState("state", "messages")] IPersistentState<AgentState<T>> state) [PersistentState("state", "messages")] IPersistentState<AgentState<T>> state, ISemanticTextMemory memory, Kernel kernel)
{ {
_state = state; _state = state;
_memory = memory;
_kernel = kernel;
} }
protected IPersistentState<AgentState<T>> _state; protected IPersistentState<AgentState<T>> _state;
private readonly ISemanticTextMemory _memory;
private readonly Kernel _kernel;
protected void AddToHistory(string message, ChatUserType userType) protected void AddToHistory(string message, ChatUserType userType)
{ {
@ -30,19 +36,33 @@ public abstract class AiAgent<T> : Agent
return string.Join("\n", _state.State.History.Select(message => $"{message.UserType}: {message.Message}")); return string.Join("\n", _state.State.History.Select(message => $"{message.UserType}: {message.Message}"));
} }
protected virtual async Task<string> CallFunction(string template, KernelArguments arguments, Kernel kernel, OpenAIPromptExecutionSettings? settings = null) protected virtual async Task<string> CallFunction(string template, KernelArguments arguments, OpenAIPromptExecutionSettings? settings = null)
{ {
var propmptSettings = (settings == null) ? new OpenAIPromptExecutionSettings { MaxTokens = 18000, Temperature = 0.8, TopP = 1 } var propmptSettings = (settings == null) ? new OpenAIPromptExecutionSettings { MaxTokens = 18000, Temperature = 0.8, TopP = 1 }
: settings; : settings;
var function = kernel.CreateFunctionFromPrompt(template, propmptSettings); var function = _kernel.CreateFunctionFromPrompt(template, propmptSettings);
var result = (await kernel.InvokeAsync(function, arguments)).ToString(); var result = (await _kernel.InvokeAsync(function, arguments)).ToString();
AddToHistory(result, ChatUserType.Agent); AddToHistory(result, ChatUserType.Agent);
return result; return result;
} }
protected async Task<T> ShareContext() /// <summary>
/// Adds knowledge to the
/// </summary>
/// <param name="instruction">The instruction string that uses the value of !index! as a placeholder to inject the data. Example:"Consider the following architectural guidelines: {waf}" </param>
/// <param name="index">Knowledge index</param>
/// <param name="arguments">The sk arguments, "input" is the argument </param>
/// <returns></returns>
protected async Task<KernelArguments> AddKnowledge(string instruction, string index, KernelArguments arguments)
{ {
return _state.State.Data; var documents = _memory.SearchAsync(index, arguments["input"].ToString(), 5);
var kbStringBuilder = new StringBuilder();
await foreach (var doc in documents)
{
kbStringBuilder.AppendLine($"{doc.Metadata.Text}");
}
arguments[index] = instruction.Replace($"!{index}!", $"{kbStringBuilder}");
return arguments;
} }
} }

View File

@ -8,7 +8,6 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" /> <PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" />
<PackageReference Include="Microsoft.Orleans.Runtime" Version="8.0.0" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" /> <PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.Orleans.Streaming" Version="8.0.0" /> <PackageReference Include="Microsoft.Orleans.Streaming" Version="8.0.0" />
</ItemGroup> </ItemGroup>

View File

@ -1,5 +1,6 @@
using Microsoft.AI.Agents.Abstractions; using Microsoft.AI.Agents.Abstractions;
using Microsoft.KernelMemory; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime; using Orleans.Runtime;
using Orleans.Streams; using Orleans.Streams;
@ -8,17 +9,16 @@ namespace Microsoft.AI.DevTeam;
// The architect has Org+Repo scope and is holding the knowledge of the high level architecture of the project // The architect has Org+Repo scope and is holding the knowledge of the high level architecture of the project
[ImplicitStreamSubscription(Consts.MainNamespace)] [ImplicitStreamSubscription(Consts.MainNamespace)]
public class Architect : AzureAiAgent<ArchitectState> public class Architect : AiAgent<ArchitectState>
{ {
protected override string Namespace => Consts.MainNamespace; protected override string Namespace => Consts.MainNamespace;
public Architect([PersistentState("state", "messages")] IPersistentState<AgentState<ArchitectState>> state, IKernelMemory memory) public Architect([PersistentState("state", "messages")] IPersistentState<AgentState<ArchitectState>> state, ISemanticTextMemory memory, Kernel kernel)
: base(state, memory) : base(state, memory, kernel)
{ {
} }
public override Task HandleEvent(Event item, StreamSequenceToken? token) public override Task HandleEvent(Event item, StreamSequenceToken? token)
{ {
// throw new NotImplementedException();
return Task.CompletedTask; return Task.CompletedTask;
} }
} }

View File

@ -1,30 +0,0 @@
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel;
using Orleans.Runtime;
using Microsoft.AI.Agents.Abstractions;
using Microsoft.SemanticKernel.Connectors.OpenAI;
namespace Microsoft.AI.DevTeam;
public abstract class AzureAiAgent<T> : AiAgent<T>
{
private readonly IKernelMemory _memory;
public AzureAiAgent([PersistentState("state", "messages")] IPersistentState<AgentState<T>> state, IKernelMemory memory) : base(state)
{
_memory = memory;
}
protected async Task<KernelArguments> AddWafContext(IKernelMemory memory, KernelArguments arguments)
{
var waf = await memory.AskAsync(arguments["input"].ToString(), index: "waf");
if (!waf.NoResult) arguments["wafContext"] = $"Consider the following architectural guidelines: ${waf.Result}";
return arguments;
}
protected override async Task<string> CallFunction(string template, KernelArguments arguments, Kernel kernel, OpenAIPromptExecutionSettings? settings = null)
{
var wafArguments = await AddWafContext(_memory, arguments);
return await base.CallFunction(template, wafArguments, kernel, settings);
}
}

View File

@ -1,23 +1,22 @@
using Microsoft.AI.Agents.Abstractions; using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events; using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime; using Orleans.Runtime;
using Orleans.Streams; using Orleans.Streams;
namespace Microsoft.AI.DevTeam; namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)] [ImplicitStreamSubscription(Consts.MainNamespace)]
public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps public class Dev : AiAgent<DeveloperState>, IDevelopApps
{ {
protected override string Namespace => Consts.MainNamespace; protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<Dev> _logger; private readonly ILogger<Dev> _logger;
public Dev([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperState>> state, Kernel kernel, IKernelMemory memory, ILogger<Dev> logger) public Dev([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<Dev> logger)
: base(state, memory) : base(state, memory, kernel)
{ {
_kernel = kernel;
_logger = logger; _logger = logger;
} }
@ -65,7 +64,9 @@ public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps
{ {
// TODO: ask the architect for the high level architecture as well as the files structure of the project // TODO: ask the architect for the high level architecture as well as the files structure of the project
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)}; var context = new KernelArguments { ["input"] = AppendChatHistory(ask)};
return await CallFunction(DeveloperSkills.Implement, context, _kernel); var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf",context);
return await CallFunction(DeveloperSkills.Implement, enhancedContext);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -73,40 +74,6 @@ public class Dev : AzureAiAgent<DeveloperState>, IDevelopApps
return default; return default;
} }
} }
// public async Task<UnderstandingResult> BuildUnderstanding(string content)
// {
// try
// {
// var explainFunction = _kernel.CreateSemanticFunction(Developer.Explain, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
// var consolidateFunction = _kernel.CreateSemanticFunction(Developer.ConsolidateUnderstanding, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
// var explainContext = new ContextVariables();
// explainContext.Set("input", content);
// var explainResult = await _kernel.RunAsync(explainContext, explainFunction);
// var explainMesage = explainResult.ToString();
// var consolidateContext = new ContextVariables();
// consolidateContext.Set("input", _state.State.Understanding);
// consolidateContext.Set("newUnderstanding", explainMesage);
// var consolidateResult = await _kernel.RunAsync(consolidateContext, consolidateFunction);
// var consolidateMessage = consolidateResult.ToString();
// _state.State.Understanding = consolidateMessage;
// await _state.WriteStateAsync();
// return new UnderstandingResult
// {
// NewUnderstanding = consolidateMessage,
// Explanation = explainMesage
// };
// }
// catch (Exception ex)
// {
// _logger.LogError(ex, "Error building understanding");
// return default;
// }
// }
} }
[GenerateSerializer] [GenerateSerializer]

View File

@ -1,23 +1,20 @@
using Microsoft.AI.Agents.Abstractions; using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events; using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI; using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime; using Orleans.Runtime;
using Orleans.Streams; using Orleans.Streams;
namespace Microsoft.AI.DevTeam; namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)] [ImplicitStreamSubscription(Consts.MainNamespace)]
public class DeveloperLead : AzureAiAgent<DeveloperLeadState>, ILeadDevelopers public class DeveloperLead : AiAgent<DeveloperLeadState>, ILeadDevelopers
{ {
protected override string Namespace => Consts.MainNamespace; protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<DeveloperLead> _logger; private readonly ILogger<DeveloperLead> _logger;
public DeveloperLead([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperLeadState>> state, Kernel kernel, IKernelMemory memory, ILogger<DeveloperLead> logger) public DeveloperLead([PersistentState("state", "messages")] IPersistentState<AgentState<DeveloperLeadState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<DeveloperLead> logger)
: base(state, memory) : base(state, memory, kernel)
{ {
_kernel = kernel;
_logger = logger; _logger = logger;
} }
@ -64,8 +61,10 @@ public class DeveloperLead : AzureAiAgent<DeveloperLeadState>, ILeadDevelopers
{ {
// TODO: Ask the architect for the existing high level architecture // TODO: Ask the architect for the existing high level architecture
// as well as the file structure // as well as the file structure
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)}; var context = new KernelArguments { ["input"] = AppendChatHistory(ask) };
return await CallFunction(DevLeadSkills.Plan, context, _kernel); var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf", context);
return await CallFunction(DevLeadSkills.Plan, enhancedContext);
} }
catch (Exception ex) catch (Exception ex)
{ {

View File

@ -1,24 +1,21 @@
using Microsoft.AI.Agents.Abstractions; using Microsoft.AI.Agents.Abstractions;
using Microsoft.AI.DevTeam.Events; using Microsoft.AI.DevTeam.Events;
using Microsoft.KernelMemory;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Memory;
using Orleans.Runtime; using Orleans.Runtime;
using Orleans.Streams; using Orleans.Streams;
namespace Microsoft.AI.DevTeam; namespace Microsoft.AI.DevTeam;
[ImplicitStreamSubscription(Consts.MainNamespace)] [ImplicitStreamSubscription(Consts.MainNamespace)]
public class ProductManager : AzureAiAgent<ProductManagerState>, IManageProducts public class ProductManager : AiAgent<ProductManagerState>, IManageProducts
{ {
protected override string Namespace => Consts.MainNamespace; protected override string Namespace => Consts.MainNamespace;
private readonly Kernel _kernel;
private readonly ILogger<ProductManager> _logger; private readonly ILogger<ProductManager> _logger;
public ProductManager([PersistentState("state", "messages")] IPersistentState<AgentState<ProductManagerState>> state, Kernel kernel, IKernelMemory memory, ILogger<ProductManager> logger) public ProductManager([PersistentState("state", "messages")] IPersistentState<AgentState<ProductManagerState>> state, Kernel kernel, ISemanticTextMemory memory, ILogger<ProductManager> logger)
: base(state, memory) : base(state, memory, kernel)
{ {
_kernel = kernel;
//_memory = memory;
_logger = logger; _logger = logger;
} }
@ -63,7 +60,9 @@ public class ProductManager : AzureAiAgent<ProductManagerState>, IManageProducts
try try
{ {
var context = new KernelArguments { ["input"] = AppendChatHistory(ask)}; var context = new KernelArguments { ["input"] = AppendChatHistory(ask)};
return await CallFunction(PMSkills.Readme, context, _kernel); var instruction = "Consider the following architectural guidelines:!waf!";
var enhancedContext = await AddKnowledge(instruction, "waf",context);
return await CallFunction(PMSkills.Readme, enhancedContext);
} }
catch (Exception ex) catch (Exception ex)
{ {

View File

@ -9,12 +9,14 @@ using Octokit.Webhooks.AspNetCore;
using Azure.Identity; using Azure.Identity;
using Microsoft.Extensions.Azure; using Microsoft.Extensions.Azure;
using Microsoft.Extensions.Http.Resilience; using Microsoft.Extensions.Http.Resilience;
using Microsoft.KernelMemory; using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Connectors.OpenAI;
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
builder.Services.AddSingleton<WebhookEventProcessor, GithubWebHookProcessor>(); builder.Services.AddSingleton<WebhookEventProcessor, GithubWebHookProcessor>();
builder.Services.AddTransient(CreateKernel); builder.Services.AddTransient(CreateKernel);
builder.Services.AddSingleton<IKernelMemory>(CreateMemory); builder.Services.AddTransient(CreateMemory);
builder.Services.AddHttpClient(); builder.Services.AddHttpClient();
builder.Services.AddSingleton(s => builder.Services.AddSingleton(s =>
@ -22,7 +24,7 @@ builder.Services.AddSingleton(s =>
var ghOptions = s.GetService<IOptions<GithubOptions>>(); var ghOptions = s.GetService<IOptions<GithubOptions>>();
var logger = s.GetService<ILogger<GithubAuthService>>(); var logger = s.GetService<ILogger<GithubAuthService>>();
var ghService = new GithubAuthService(ghOptions, logger); var ghService = new GithubAuthService(ghOptions, logger);
var client = ghService.GetGitHubClient().Result; var client = ghService.GetGitHubClient();
return client; return client;
}); });
@ -96,29 +98,24 @@ app.Map("/dashboard", x => x.UseOrleansDashboard());
app.Run(); app.Run();
static IKernelMemory CreateMemory(IServiceProvider provider) static ISemanticTextMemory CreateMemory(IServiceProvider provider)
{ {
var qdrantConfig = provider.GetService<IOptions<QdrantOptions>>().Value;
var openAiConfig = provider.GetService<IOptions<OpenAIOptions>>().Value; var openAiConfig = provider.GetService<IOptions<OpenAIOptions>>().Value;
return new KernelMemoryBuilder() var qdrantConfig = provider.GetService<IOptions<QdrantOptions>>().Value;
.WithQdrantMemoryDb(qdrantConfig.Endpoint)
.WithAzureOpenAITextGeneration(new AzureOpenAIConfig var loggerFactory = LoggerFactory.Create(builder =>
{ {
APIType = AzureOpenAIConfig.APITypes.ChatCompletion, builder
Endpoint = openAiConfig.Endpoint, .SetMinimumLevel(LogLevel.Debug)
Deployment = openAiConfig.DeploymentOrModelId, .AddConsole()
Auth = AzureOpenAIConfig.AuthTypes.APIKey, .AddDebug();
APIKey = openAiConfig.ApiKey });
})
.WithAzureOpenAITextEmbeddingGeneration(new AzureOpenAIConfig var memoryBuilder = new MemoryBuilder();
{ return memoryBuilder.WithLoggerFactory(loggerFactory)
APIType = AzureOpenAIConfig.APITypes.EmbeddingGeneration, .WithQdrantMemoryStore(qdrantConfig.Endpoint, qdrantConfig.VectorSize)
Endpoint = openAiConfig.Endpoint, .WithAzureOpenAITextEmbeddingGeneration(openAiConfig.EmbeddingDeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey)
Deployment =openAiConfig.EmbeddingDeploymentOrModelId, .Build();
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = openAiConfig.ApiKey
})
.Build<MemoryServerless>();
} }
static Kernel CreateKernel(IServiceProvider provider) static Kernel CreateKernel(IServiceProvider provider)

View File

@ -44,7 +44,7 @@ public class GithubAuthService
return new JwtSecurityTokenHandler().WriteToken(token); return new JwtSecurityTokenHandler().WriteToken(token);
} }
public async Task<GitHubClient> GetGitHubClient() public GitHubClient GetGitHubClient()
{ {
try try
{ {
@ -53,7 +53,7 @@ public class GithubAuthService
{ {
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer) Credentials = new Credentials(jwtToken, AuthenticationType.Bearer)
}; };
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId); var response = appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId).Result;
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}")) return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{ {
Credentials = new Credentials(response.Token) Credentials = new Credentials(response.Token)

View File

@ -21,7 +21,6 @@
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" /> <PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.6.2-alpha" /> <PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.6.2-alpha" /> <PackageReference Include="Microsoft.SemanticKernel.Plugins.Memory" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240318.1" />
<PackageReference Include="Microsoft.Orleans.Server" Version="8.0.0" /> <PackageReference Include="Microsoft.Orleans.Server" Version="8.0.0" />
<PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" /> <PackageReference Include="Microsoft.Orleans.Sdk" Version="8.0.0" />
@ -45,7 +44,7 @@
<PackageReference Include="Azure.Identity" Version="1.11.0-beta.1" /> <PackageReference Include="Azure.Identity" Version="1.11.0-beta.1" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.4.1" /> <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.4.1" />
<PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240318.1" />
</ItemGroup> </ItemGroup>

View File

@ -1,5 +1,10 @@
using Microsoft.Extensions.Logging; using System.Reflection;
using Microsoft.KernelMemory; using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Connectors.Qdrant;
using Microsoft.SemanticKernel.Memory;
using UglyToad.PdfPig;
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
class Program class Program
{ {
@ -16,33 +21,37 @@ class Program
.AddDebug(); .AddDebug();
}); });
var memory = new KernelMemoryBuilder() var memoryBuilder = new MemoryBuilder();
.WithQdrantMemoryDb(kernelSettings.QdrantEndpoint) var memory = memoryBuilder.WithLoggerFactory(loggerFactory)
.WithAzureOpenAITextGeneration(new AzureOpenAIConfig .WithQdrantMemoryStore(kernelSettings.QdrantEndpoint, 1536)
{ .WithAzureOpenAITextEmbeddingGeneration(kernelSettings.EmbeddingDeploymentOrModelId,kernelSettings.Endpoint, kernelSettings.ApiKey)
APIType = AzureOpenAIConfig.APITypes.ChatCompletion, .Build();
Endpoint =kernelSettings.Endpoint,
Deployment = kernelSettings.DeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = kernelSettings.ApiKey
})
.WithAzureOpenAITextEmbeddingGeneration(new AzureOpenAIConfig
{
APIType = AzureOpenAIConfig.APITypes.EmbeddingGeneration,
Endpoint = kernelSettings.Endpoint,
Deployment =kernelSettings.EmbeddingDeploymentOrModelId,
Auth = AzureOpenAIConfig.AuthTypes.APIKey,
APIKey = kernelSettings.ApiKey
})
.Build<MemoryServerless>();
await ImportDocumentAsync(memory, WafFileName); await ImportDocumentAsync(memory, WafFileName);
} }
public static async Task ImportDocumentAsync(IKernelMemory memory, string filename) public static async Task ImportDocumentAsync(ISemanticTextMemory memory, string filename)
{ {
await memory.ImportDocumentAsync(new Document("wafdoc") var currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
.AddFiles([ var filePath = Path.Combine(currentDirectory, filename);
filename using var pdfDocument = PdfDocument.Open(File.OpenRead(filePath));
]), index: "waf"); var pages = pdfDocument.GetPages();
foreach (var page in pages)
{
try
{
var text = ContentOrderTextExtractor.GetText(page);
var descr = text.Take(100);
await memory.SaveInformationAsync(
collection: "waf",
text: text,
id: $"{Guid.NewGuid()}",
description: $"Document: {descr}");
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
}
}
} }
} }

View File

@ -72,7 +72,7 @@ internal class KernelSettings
} }
var configuration = new ConfigurationBuilder() var configuration = new ConfigurationBuilder()
.SetBasePath(System.IO.Directory.GetCurrentDirectory()) .SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile(configFile, optional: true, reloadOnChange: true) .AddJsonFile(configFile, optional: true, reloadOnChange: true)
.AddEnvironmentVariables() .AddEnvironmentVariables()
.Build(); .Build();

View File

@ -11,8 +11,14 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" /> <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" /> <PackageReference Include="Microsoft.SemanticKernel" Version="1.6.2" />
<PackageReference Include="Microsoft.KernelMemory.MemoryDb.Qdrant" Version="0.35.240318.1" /> <PackageReference Include="Microsoft.SemanticKernel.Connectors.Qdrant" Version="1.6.2-alpha" />
<PackageReference Include="Microsoft.KernelMemory.Core" Version="0.35.240318.1" /> <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
<PackageReference Include="PdfPig" Version="0.1.9-alpha-20240324-e7896" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>