2023-06-15 13:57:47 +02:00
|
|
|
|
using UglyToad.PdfPig;
|
|
|
|
|
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
|
|
|
|
|
using Microsoft.SemanticKernel;
|
|
|
|
|
using Microsoft.SemanticKernel.Text;
|
|
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
|
using System.Text;
|
|
|
|
|
using Microsoft.SemanticKernel.Connectors.Memory.Qdrant;
|
|
|
|
|
using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
|
|
|
|
|
using Microsoft.SemanticKernel.Memory;
|
|
|
|
|
using System.Reflection;
|
|
|
|
|
|
|
|
|
|
class Program
|
|
|
|
|
{
|
|
|
|
|
static string WafFileName = "azure-well-architected.pdf";
|
|
|
|
|
static async Task Main(string[] args)
|
|
|
|
|
{
|
|
|
|
|
var kernelSettings = KernelSettings.LoadSettings();
|
|
|
|
|
var kernelConfig = new KernelConfig();
|
|
|
|
|
|
|
|
|
|
using ILoggerFactory loggerFactory = LoggerFactory.Create(builder =>
|
|
|
|
|
{
|
|
|
|
|
builder
|
|
|
|
|
.SetMinimumLevel(kernelSettings.LogLevel ?? LogLevel.Warning)
|
|
|
|
|
.AddConsole()
|
|
|
|
|
.AddDebug();
|
|
|
|
|
});
|
|
|
|
|
|
2023-08-28 20:57:56 +02:00
|
|
|
|
var memoryStore = new QdrantMemoryStore(new QdrantVectorDbClient(kernelSettings.QdrantEndpoint, 1536));
|
2023-06-15 13:57:47 +02:00
|
|
|
|
var embedingGeneration = new AzureTextEmbeddingGeneration(kernelSettings.EmbeddingDeploymentOrModelId, kernelSettings.Endpoint, kernelSettings.ApiKey);
|
|
|
|
|
var semanticTextMemory = new SemanticTextMemory(memoryStore, embedingGeneration);
|
|
|
|
|
|
|
|
|
|
var kernel = new KernelBuilder()
|
|
|
|
|
.WithLogger(loggerFactory.CreateLogger<IKernel>())
|
|
|
|
|
.WithAzureChatCompletionService(kernelSettings.DeploymentOrModelId, kernelSettings.Endpoint, kernelSettings.ApiKey, true, kernelSettings.ServiceId, true)
|
|
|
|
|
.WithMemory(semanticTextMemory)
|
|
|
|
|
.WithConfiguration(kernelConfig).Build();
|
|
|
|
|
await ImportDocumentAsync(kernel, WafFileName);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static async Task ImportDocumentAsync(IKernel kernel, string filename)
|
|
|
|
|
{
|
|
|
|
|
var currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
|
|
|
|
|
var filePath = Path.Combine(currentDirectory, filename);
|
|
|
|
|
using var pdfDocument = PdfDocument.Open(File.OpenRead(filePath));
|
|
|
|
|
var pages = pdfDocument.GetPages();
|
|
|
|
|
foreach (var page in pages)
|
|
|
|
|
{
|
2023-08-28 20:57:56 +02:00
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var text = ContentOrderTextExtractor.GetText(page);
|
|
|
|
|
var descr = text.Take(100);
|
|
|
|
|
await kernel.Memory.SaveInformationAsync(
|
|
|
|
|
collection: "waf-pages",
|
|
|
|
|
text: text,
|
|
|
|
|
id: $"{Guid.NewGuid()}",
|
|
|
|
|
description: $"Document: {descr}");
|
|
|
|
|
}
|
|
|
|
|
catch(Exception ex)
|
|
|
|
|
{
|
|
|
|
|
Console.WriteLine(ex.Message);
|
|
|
|
|
}
|
2023-06-15 13:57:47 +02:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|