add error handling/logging

This commit is contained in:
Kosta Petan 2023-11-09 16:03:38 +00:00
parent d18296f800
commit 45d9282ab2
11 changed files with 420 additions and 269 deletions

View File

@ -5,6 +5,7 @@ using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant;
using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Reliability.Basic;
using Octokit.Webhooks; using Octokit.Webhooks;
using Octokit.Webhooks.AspNetCore; using Octokit.Webhooks.AspNetCore;
using Orleans.Configuration; using Orleans.Configuration;
@ -17,7 +18,8 @@ builder.Services.AddHttpClient();
builder.Services.AddSingleton(s => builder.Services.AddSingleton(s =>
{ {
var ghOptions = s.GetService<IOptions<GithubOptions>>(); var ghOptions = s.GetService<IOptions<GithubOptions>>();
var ghService = new GithubAuthService(ghOptions); var logger = s.GetService<ILogger<GithubAuthService>>();
var ghService = new GithubAuthService(ghOptions, logger);
var client = ghService.GetGitHubClient().Result; var client = ghService.GetGitHubClient().Result;
return client; return client;
}); });
@ -160,5 +162,9 @@ static IKernel CreateKernel(IServiceProvider provider)
return new KernelBuilder() return new KernelBuilder()
.WithLoggerFactory(loggerFactory) .WithLoggerFactory(loggerFactory)
.WithAzureChatCompletionService(openAiConfig.DeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey, true, openAiConfig.ServiceId, true) .WithAzureChatCompletionService(openAiConfig.DeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey, true, openAiConfig.ServiceId, true)
.WithRetryBasic(new BasicRetryConfig {
MaxRetryCount = 5,
UseExponentialBackoff = true
})
.WithMemory(semanticTextMemory).Build(); .WithMemory(semanticTextMemory).Build();
} }

View File

@ -211,7 +211,7 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor
Org = org, Org = org,
Repo = repo, Repo = repo,
Number = (int)issueNumber, Number = (int)issueNumber,
Content = readme Content = string.IsNullOrEmpty(readme)? "Sorry, something went wrong": readme
}); });
} }
else if (skillName == nameof(DevLead) && functionName == nameof(DevLead.Plan)) else if (skillName == nameof(DevLead) && functionName == nameof(DevLead.Plan))
@ -223,19 +223,20 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor
Org = org, Org = org,
Repo = repo, Repo = repo,
Number = (int)issueNumber, Number = (int)issueNumber,
Content = plan Content = string.IsNullOrEmpty(plan)? "Sorry, something went wrong":plan
}); });
} }
else if (skillName == nameof(Developer) && functionName == nameof(Developer.Implement)) else if (skillName == nameof(Developer) && functionName == nameof(Developer.Implement))
{ {
var dev = _grains.GetGrain<IDevelopCode>(issueNumber, suffix); var dev = _grains.GetGrain<IDevelopCode>(issueNumber, suffix);
var code = await dev.GenerateCode(input); var code = await dev.GenerateCode(input);
await _ghService.PostComment(new PostCommentRequest await _ghService.PostComment(new PostCommentRequest
{ {
Org = org, Org = org,
Repo = repo, Repo = repo,
Number = (int)issueNumber, Number = (int)issueNumber,
Content = code Content = string.IsNullOrEmpty(code)? "Sorry, something went wrong":code
}); });
} }
else { }// something went wrong else { }// something went wrong

View File

@ -37,8 +37,6 @@ public class Conductor : Grain, IOrchestrateWorkflows
ParentNumber = parentNumber ParentNumber = parentNumber
}); });
var suffix = $"{org}-{repo}"; var suffix = $"{org}-{repo}";
var pm = GrainFactory.GetGrain<IManageProduct>(pmIssue.IssueNumber, suffix);
var devLead = GrainFactory.GetGrain<ILeadDevelopment>(devLeadIssue.IssueNumber, suffix);
var lookup = GrainFactory.GetGrain<ILookupMetadata>(suffix); var lookup = GrainFactory.GetGrain<ILookupMetadata>(suffix);
var metadataList = new List<StoreMetadataPairs>{ var metadataList = new List<StoreMetadataPairs>{
@ -81,9 +79,4 @@ public class Conductor : Grain, IOrchestrateWorkflows
} }
await lookup.StoreMetadata(metadataList); await lookup.StoreMetadata(metadataList);
} }
public Task ScheduleCommitSandboxRun(CommitRequest commitRequest, MarkTaskCompleteRequest markTaskCompleteRequest)
{
throw new NotImplementedException();
}
} }

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills; using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration;
@ -9,44 +10,49 @@ namespace Microsoft.AI.DevTeam;
public class DeveloperLead : SemanticPersona, ILeadDevelopment public class DeveloperLead : SemanticPersona, ILeadDevelopment
{ {
private readonly IKernel _kernel; private readonly IKernel _kernel;
private readonly ILogger<DeveloperLead> _logger;
protected override string MemorySegment => "dev-lead-memory"; protected override string MemorySegment => "dev-lead-memory";
public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state) : base(state) public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<DeveloperLead> logger) : base(state)
{ {
_kernel = kernel; _kernel = kernel;
_logger = logger;
} }
public async Task<string> CreatePlan(string ask) public async Task<string> CreatePlan(string ask)
{ {
// var architectId = Guid.NewGuid(); try
// var plan = "this is my plan";
// var architect = GrainFactory.GetGrain<IArchitectSolutions>(architectId);
// var review = architect.ReviewPlan(plan);
// return Task.FromResult(plan);
var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 });
var context = new ContextVariables();
context.Set("input", ask);
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{ {
Message = ask, var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 });
Order = _state.State.History.Count + 1, var context = new ContextVariables();
UserType = ChatUserType.User context.Set("input", ask);
}); if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
await AddWafContext(_kernel, ask, context); _state.State.History.Add(new ChatHistoryItem
context.Set("input", ask); {
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function); var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString(); var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem _state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch (Exception ex)
{ {
Message = resultMessage, _logger.LogError(ex, "Error creating development plan");
Order = _state.State.History.Count + 1, return default;
UserType = ChatUserType.Agent }
});
await _state.WriteStateAsync();
return resultMessage;
} }
public Task<DevLeadPlanResponse> GetLatestPlan() public Task<DevLeadPlanResponse> GetLatestPlan()
@ -55,11 +61,6 @@ public class DeveloperLead : SemanticPersona, ILeadDevelopment
var response = JsonSerializer.Deserialize<DevLeadPlanResponse>(plan); var response = JsonSerializer.Deserialize<DevLeadPlanResponse>(plan);
return Task.FromResult(response); return Task.FromResult(response);
} }
public Task<string> BuildUnderstanding(string content)
{
throw new NotImplementedException();
}
} }
[GenerateSerializer] [GenerateSerializer]

View File

@ -1,6 +1,6 @@
namespace Microsoft.AI.DevTeam; namespace Microsoft.AI.DevTeam;
public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory, IUnderstand public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory
{ {
Task<string> CreatePlan(string ask); Task<string> CreatePlan(string ask);
Task<DevLeadPlanResponse> GetLatestPlan(); Task<DevLeadPlanResponse> GetLatestPlan();

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills; using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration;
@ -9,38 +10,49 @@ namespace Microsoft.AI.DevTeam;
public class Dev : SemanticPersona, IDevelopCode public class Dev : SemanticPersona, IDevelopCode
{ {
private readonly IKernel _kernel; private readonly IKernel _kernel;
private readonly ILogger<Dev> _logger;
protected override string MemorySegment => "dev-memory"; protected override string MemorySegment => "dev-memory";
public Dev(IKernel kernel, [PersistentState("state", "messages")]IPersistentState<SemanticPersonaState> state) : base(state) public Dev(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<Dev> logger) : base(state)
{ {
_kernel = kernel; _kernel = kernel;
_logger = logger;
} }
public async Task<string> GenerateCode(string ask) public async Task<string> GenerateCode(string ask)
{ {
var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 }); try
var context = new ContextVariables();
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{ {
Message = ask, var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
Order = _state.State.History.Count + 1, var context = new ContextVariables();
UserType = ChatUserType.User if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
}); _state.State.History.Add(new ChatHistoryItem
await AddWafContext(_kernel, ask, context); {
context.Set("input", ask); Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function); var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString(); var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem _state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch(Exception ex)
{ {
Message = resultMessage, _logger.LogError(ex, "Error generating code");
Order = _state.State.History.Count + 1, return default;
UserType = ChatUserType.Agent }
});
await _state.WriteStateAsync();
return resultMessage;
} }

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills; using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel; using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.Orchestration;
@ -8,36 +9,47 @@ namespace Microsoft.AI.DevTeam;
public class ProductManager : SemanticPersona, IManageProduct public class ProductManager : SemanticPersona, IManageProduct
{ {
private readonly IKernel _kernel; private readonly IKernel _kernel;
private readonly ILogger<ProductManager> _logger;
protected override string MemorySegment => "pm-memory"; protected override string MemorySegment => "pm-memory";
public ProductManager(IKernel kernel,[PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state) : base(state) public ProductManager(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<ProductManager> logger) : base(state)
{ {
_kernel = kernel; _kernel = kernel;
_logger = logger;
} }
public async Task<string> CreateReadme(string ask) public async Task<string> CreateReadme(string ask)
{ {
var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 }); try
var context = new ContextVariables();
context.Set("input", ask);
if(_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{ {
Message = ask, var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 });
Order = _state.State.History.Count + 1, var context = new ContextVariables();
UserType = ChatUserType.User context.Set("input", ask);
}); if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
await AddWafContext(_kernel, ask, context); _state.State.History.Add(new ChatHistoryItem
context.Set("input", ask); {
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function); var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString(); var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem _state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch (Exception ex)
{ {
Message = resultMessage, _logger.LogError(ex, "Error creating readme");
Order = _state.State.History.Count + 1, return default;
UserType = ChatUserType.Agent }
});
await _state.WriteStateAsync();
return resultMessage;
} }
} }

View File

@ -7,6 +7,7 @@ using Azure.ResourceManager.ContainerInstance;
using Azure.ResourceManager.ContainerInstance.Models; using Azure.ResourceManager.ContainerInstance.Models;
using Azure.ResourceManager.Resources; using Azure.ResourceManager.Resources;
using Azure.Storage.Files.Shares; using Azure.Storage.Files.Shares;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@ -15,108 +16,142 @@ namespace Microsoft.AI.DevTeam;
public class AzureService : IManageAzure public class AzureService : IManageAzure
{ {
private readonly AzureOptions _azSettings; private readonly AzureOptions _azSettings;
private readonly ILogger<AzureService> _logger;
public AzureService(IOptions<AzureOptions> azOptions) public AzureService(IOptions<AzureOptions> azOptions, ILogger<AzureService> logger)
{ {
_azSettings = azOptions.Value; _azSettings = azOptions.Value;
_logger = logger;
} }
public async Task DeleteSandbox(string sandboxId) public async Task DeleteSandbox(string sandboxId)
{ {
var client = new ArmClient(new DefaultAzureCredential()); try
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); {
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
await containerGroup.Value.DeleteAsync(WaitUntil.Started);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting sandbox");
}
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
await containerGroup.Value.DeleteAsync(WaitUntil.Started);
} }
public async Task<bool> IsSandboxCompleted(string sandboxId) public async Task<bool> IsSandboxCompleted(string sandboxId)
{ {
var client = new ArmClient(new DefaultAzureCredential()); try
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); {
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var collection = resourceGroupResource.GetContainerGroups(); var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId); var containerGroup = await collection.GetAsync(sandboxId);
return containerGroup.Value.Data.ProvisioningState == "Succeeded" return containerGroup.Value.Data.ProvisioningState == "Succeeded"
&& containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated"; && containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated";
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking sandbox status");
return false;
}
} }
public async Task RunInSandbox(SandboxRequest request) public async Task RunInSandbox(SandboxRequest request)
{ {
var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ? try
{
var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ?
new ArmClient(new AzureCliCredential()) new ArmClient(new AzureCliCredential())
: new ArmClient(new ManagedIdentityCredential(_azSettings.ManagedIdentity)); : new ArmClient(new ManagedIdentityCredential(_azSettings.ManagedIdentity));
var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}"; var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}";
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh"; var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh";
var collection = resourceGroupResource.GetContainerGroups(); var collection = resourceGroupResource.GetContainerGroups();
var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[] var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[]
{ {
new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1))) new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1)))
{
Command = { "/bin/bash", $"{scriptPath}" },
VolumeMounts =
{ {
new ContainerVolumeMount("azfiles","/azfiles/") Command = { "/bin/bash", $"{scriptPath}" },
VolumeMounts =
{ {
IsReadOnly = false, new ContainerVolumeMount("azfiles","/azfiles/")
} {
}, IsReadOnly = false,
}}, ContainerInstanceOperatingSystemType.Linux) }
{ },
Volumes = }}, ContainerInstanceOperatingSystemType.Linux)
{ {
new ContainerVolume("azfiles") Volumes =
{ {
AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName) new ContainerVolume("azfiles")
{ {
StorageAccountKey = _azSettings.FilesAccountKey AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName)
{
StorageAccountKey = _azSettings.FilesAccountKey
},
}, },
}, },
}, RestartPolicy = ContainerGroupRestartPolicy.Never,
RestartPolicy = ContainerGroupRestartPolicy.Never, Sku = ContainerGroupSku.Standard,
Sku = ContainerGroupSku.Standard, Priority = ContainerGroupPriority.Regular
Priority = ContainerGroupPriority.Regular };
}; await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data);
await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data); }
catch (Exception ex)
{
_logger.LogError(ex, "Error running sandbox");
}
} }
public async Task Store(SaveOutputRequest request) public async Task Store(SaveOutputRequest request)
{ {
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; try
var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}";
var fileName = $"{request.FileName}.{request.Extension}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
await share.CreateIfNotExistsAsync();
await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ;
var parentDir = share.GetDirectoryClient(parentDirName);
await parentDir.CreateIfNotExistsAsync();
var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}");
await parentIssueDir.CreateIfNotExistsAsync();
var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}");
await directory.CreateIfNotExistsAsync();
var file = directory.GetFileClient(fileName);
// hack to enable script to save files in the same directory
var cwdHack = "#!/bin/bash\n cd $(dirname $0)";
var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output)))
{ {
await file.CreateAsync(stream.Length); var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
await file.UploadRangeAsync( var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}";
new HttpRange(0, stream.Length),
stream); var fileName = $"{request.FileName}.{request.Extension}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
await share.CreateIfNotExistsAsync();
await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ;
var parentDir = share.GetDirectoryClient(parentDirName);
await parentDir.CreateIfNotExistsAsync();
var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}");
await parentIssueDir.CreateIfNotExistsAsync();
var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}");
await directory.CreateIfNotExistsAsync();
var file = directory.GetFileClient(fileName);
// hack to enable script to save files in the same directory
var cwdHack = "#!/bin/bash\n cd $(dirname $0)";
var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output)))
{
await file.CreateAsync(stream.Length);
await file.UploadRangeAsync(
new HttpRange(0, stream.Length),
stream);
}
} }
catch (Exception ex)
{
_logger.LogError(ex, "Error storing output");
}
} }
} }

View File

@ -1,6 +1,7 @@
using System.Net.Http.Json; using System.Net.Http.Json;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Octokit.Internal; using Octokit.Internal;
@ -14,22 +15,32 @@ public class CodeAnalyzer : IAnalyzeCode
{ {
private readonly ServiceOptions _serviceOptions; private readonly ServiceOptions _serviceOptions;
private readonly HttpClient _httpClient; private readonly HttpClient _httpClient;
private readonly ILogger<CodeAnalyzer> _logger;
public CodeAnalyzer(IOptions<ServiceOptions> serviceOptions, HttpClient httpClient) public CodeAnalyzer(IOptions<ServiceOptions> serviceOptions, HttpClient httpClient, ILogger<CodeAnalyzer> logger)
{ {
_serviceOptions = serviceOptions.Value; _serviceOptions = serviceOptions.Value;
_httpClient = httpClient; _httpClient = httpClient;
_httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl); _logger = logger;
_httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl);
} }
public async Task<IEnumerable<CodeAnalysis>> Analyze(string content) public async Task<IEnumerable<CodeAnalysis>> Analyze(string content)
{ {
var request = new CodeAnalysisRequest { Content = content }; try
var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json"); {
var response = await _httpClient.PostAsync("api/AnalyzeCode", body); var request = new CodeAnalysisRequest { Content = content };
var stringResult = await response.Content.ReadAsStringAsync(); var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
var result = JsonSerializer.Deserialize<IEnumerable<CodeAnalysis>>(stringResult); var response = await _httpClient.PostAsync("api/AnalyzeCode", body);
return result; var stringResult = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<IEnumerable<CodeAnalysis>>(stringResult);
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error analyzing code");
return Enumerable.Empty<CodeAnalysis>();
}
} }
} }

View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Octokit; using Octokit;
@ -5,32 +6,42 @@ namespace Microsoft.AI.DevTeam;
public class GithubAuthService public class GithubAuthService
{ {
private readonly GithubOptions _githubSettings; private readonly GithubOptions _githubSettings;
private readonly ILogger<GithubAuthService> _logger;
public GithubAuthService(IOptions<GithubOptions> ghOptions) public GithubAuthService(IOptions<GithubOptions> ghOptions, ILogger<GithubAuthService> logger)
{ {
_githubSettings = ghOptions.Value; _githubSettings = ghOptions.Value;
_logger = logger;
} }
public async Task<GitHubClient> GetGitHubClient() public async Task<GitHubClient> GetGitHubClient()
{ {
// Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file try
var generator = new GitHubJwt.GitHubJwtFactory( {
new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey), // Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file
new GitHubJwt.GitHubJwtFactoryOptions var generator = new GitHubJwt.GitHubJwtFactory(
{ new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey),
AppIntegrationId = _githubSettings.AppId, // The GitHub App Id new GitHubJwt.GitHubJwtFactoryOptions
ExpirationSeconds = 600 // 10 minutes is the maximum time allowed {
} AppIntegrationId = _githubSettings.AppId, // The GitHub App Id
); ExpirationSeconds = 600 // 10 minutes is the maximum time allowed
}
);
var jwtToken = generator.CreateEncodedJwtToken(); var jwtToken = generator.CreateEncodedJwtToken();
var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP")) var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP"))
{
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer)
};
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId);
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{
Credentials = new Credentials(response.Token)
};
}
catch (Exception ex)
{ {
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer) _logger.LogError(ex, "Error getting GitHub client");
}; return default;
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId); }
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{
Credentials = new Credentials(response.Token)
};
} }
} }

View File

@ -25,126 +25,195 @@ public class GithubService : IManageGithub
public async Task CommitToBranch(CommitRequest request) public async Task CommitToBranch(CommitRequest request)
{ {
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; try
var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
var directory = share.GetDirectoryClient(dirName);
var remaining = new Queue<ShareDirectoryClient>();
remaining.Enqueue(directory);
while (remaining.Count > 0)
{ {
var dir = remaining.Dequeue(); var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
await foreach (var item in dir.GetFilesAndDirectoriesAsync())
{
if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR
{
try
{
var file = dir.GetFileClient(item.Name);
var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "")
.Replace($"{dirName}/", "");
var fileStream = await file.OpenReadAsync();
using (var reader = new StreamReader(fileStream, Encoding.UTF8))
{
var value = reader.ReadToEnd();
await _ghClient.Repository.Content.CreateFile( var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}";
request.Org, request.Repo, filePath, var share = new ShareClient(connectionString, _azSettings.FilesShareName);
new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message var directory = share.GetDirectoryClient(dirName);
var remaining = new Queue<ShareDirectoryClient>();
remaining.Enqueue(directory);
while (remaining.Count > 0)
{
var dir = remaining.Dequeue();
await foreach (var item in dir.GetFilesAndDirectoriesAsync())
{
if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR
{
try
{
var file = dir.GetFileClient(item.Name);
var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "")
.Replace($"{dirName}/", "");
var fileStream = await file.OpenReadAsync();
using (var reader = new StreamReader(fileStream, Encoding.UTF8))
{
var value = reader.ReadToEnd();
await _ghClient.Repository.Content.CreateFile(
request.Org, request.Repo, filePath,
new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error while uploading file {item.Name}");
} }
} }
catch (Exception ex) else if (item.IsDirectory)
{ {
_logger.LogError(ex, $"Error while uploading file {item.Name}"); remaining.Enqueue(dir.GetSubdirectoryClient(item.Name));
} }
} }
else if (item.IsDirectory)
{
remaining.Enqueue(dir.GetSubdirectoryClient(item.Name));
}
} }
} }
catch (Exception ex)
{
_logger.LogError(ex, "Error committing to branch");
}
} }
public async Task CreateBranch(CreateBranchRequest request) public async Task CreateBranch(CreateBranchRequest request)
{ {
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); try
await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch); {
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error creating branch");
}
} }
public async Task<string> GetMainLanguage(string org, string repo) public async Task<string> GetMainLanguage(string org, string repo)
{ {
var languages = await _ghClient.Repository.GetAllLanguages(org, repo); try
var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First(); {
return mainLanguage.Name; var languages = await _ghClient.Repository.GetAllLanguages(org, repo);
var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First();
return mainLanguage.Name;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting main language");
return default;
}
} }
public async Task<NewIssueResponse> CreateIssue(CreateIssueRequest request) public async Task<NewIssueResponse> CreateIssue(CreateIssueRequest request)
{ {
var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}") try
{ {
Body = request.Input, var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}")
{
Body = request.Input,
}; };
newIssue.Labels.Add(request.Label); newIssue.Labels.Add(request.Label);
var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue); var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue);
var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}"; var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}";
var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody); var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody);
return new NewIssueResponse return new NewIssueResponse
{
IssueNumber = issue.Number,
CommentId = comment.Id
};
}
catch (Exception ex)
{ {
IssueNumber = issue.Number, _logger.LogError(ex, "Error creating issue");
CommentId = comment.Id return default;
}; }
} }
public async Task CreatePR(CreatePRRequest request) public async Task CreatePR(CreatePRRequest request)
{ {
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); try
await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch)); {
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch));
}
catch (Exception ex)
{
_logger.LogError(ex, "Error creating PR");
}
} }
public async Task MarkTaskComplete(MarkTaskCompleteRequest request) public async Task MarkTaskComplete(MarkTaskCompleteRequest request)
{ {
var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId); try
var updatedComment = comment.Body.Replace("[ ]", "[x]"); {
await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment); var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId);
var updatedComment = comment.Body.Replace("[ ]", "[x]");
await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error marking task complete");
}
} }
public async Task PostComment(PostCommentRequest request) public async Task PostComment(PostCommentRequest request)
{ {
await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content); try
}
public async Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent,bool> filter)
{
var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch);
return await CollectFiles(org, repo, branch, items, filter);
}
private async Task<IEnumerable<FileResponse>> CollectFiles(string org, string repo, string branch, IReadOnlyList<RepositoryContent> items, Func<RepositoryContent,bool> filter)
{
var result = new List<FileResponse>();
foreach(var item in items)
{ {
if (item.Type == ContentType.File && filter(item)) await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content);
{ }
var content = await _httpClient.GetStringAsync(item.DownloadUrl); catch (Exception ex)
result.Add(new FileResponse {
{ _logger.LogError(ex, "Error posting comment");
Name = item.Name, }
Content = content
}); }
}
else if (item.Type == ContentType.Dir) public async Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent, bool> filter)
{ {
var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo,item.Path, branch); try
result.AddRange(await CollectFiles(org, repo, branch, subItems, filter)); {
} var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch);
return await CollectFiles(org, repo, branch, items, filter);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting files");
return Enumerable.Empty<FileResponse>();
}
}
private async Task<IEnumerable<FileResponse>> CollectFiles(string org, string repo, string branch, IReadOnlyList<RepositoryContent> items, Func<RepositoryContent, bool> filter)
{
try
{
var result = new List<FileResponse>();
foreach (var item in items)
{
if (item.Type == ContentType.File && filter(item))
{
var content = await _httpClient.GetStringAsync(item.DownloadUrl);
result.Add(new FileResponse
{
Name = item.Name,
Content = content
});
}
else if (item.Type == ContentType.Dir)
{
var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, item.Path, branch);
result.AddRange(await CollectFiles(org, repo, branch, subItems, filter));
}
}
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error collecting files");
return Enumerable.Empty<FileResponse>();
} }
return result;
} }
} }
@ -164,7 +233,7 @@ public interface IManageGithub
Task CommitToBranch(CommitRequest request); Task CommitToBranch(CommitRequest request);
Task PostComment(PostCommentRequest request); Task PostComment(PostCommentRequest request);
Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent,bool> filter); Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent, bool> filter);
Task<string> GetMainLanguage(string org, string repo); Task<string> GetMainLanguage(string org, string repo);
} }