From 45d9282ab29ab9bd3b4a4fa2f81fa7c8faafb4d7 Mon Sep 17 00:00:00 2001 From: Kosta Petan Date: Thu, 9 Nov 2023 16:03:38 +0000 Subject: [PATCH] add error handling/logging --- src/apps/gh-flow/Program.cs | 8 +- .../Services/GithubWebHookProcessor.cs | 7 +- .../Actors/Conductor/Conductor.cs | 7 - .../Actors/DevLead/DeveloperLead.cs | 67 ++--- .../Actors/DevLead/ILeadDevelopment.cs | 2 +- .../Actors/Developer/Developer.cs | 58 +++-- .../Actors/ProductManager/ProductManager.cs | 54 ++-- .../Services/AzureService.cs | 175 +++++++------ .../Services/CodeAnalyzer.cs | 27 +- .../Services/GithubAuthService.cs | 49 ++-- .../Services/GithubService.cs | 235 +++++++++++------- 11 files changed, 420 insertions(+), 269 deletions(-) diff --git a/src/apps/gh-flow/Program.cs b/src/apps/gh-flow/Program.cs index 1a7418828..98f866bbf 100644 --- a/src/apps/gh-flow/Program.cs +++ b/src/apps/gh-flow/Program.cs @@ -5,6 +5,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding; using Microsoft.SemanticKernel.Connectors.Memory.Qdrant; using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Reliability.Basic; using Octokit.Webhooks; using Octokit.Webhooks.AspNetCore; using Orleans.Configuration; @@ -17,7 +18,8 @@ builder.Services.AddHttpClient(); builder.Services.AddSingleton(s => { var ghOptions = s.GetService>(); - var ghService = new GithubAuthService(ghOptions); + var logger = s.GetService>(); + var ghService = new GithubAuthService(ghOptions, logger); var client = ghService.GetGitHubClient().Result; return client; }); @@ -160,5 +162,9 @@ static IKernel CreateKernel(IServiceProvider provider) return new KernelBuilder() .WithLoggerFactory(loggerFactory) .WithAzureChatCompletionService(openAiConfig.DeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey, true, openAiConfig.ServiceId, true) + .WithRetryBasic(new BasicRetryConfig { + MaxRetryCount = 5, + UseExponentialBackoff = true + }) .WithMemory(semanticTextMemory).Build(); } \ No newline at end of file diff --git a/src/apps/gh-flow/Services/GithubWebHookProcessor.cs b/src/apps/gh-flow/Services/GithubWebHookProcessor.cs index 708e7bff1..d223dc380 100644 --- a/src/apps/gh-flow/Services/GithubWebHookProcessor.cs +++ b/src/apps/gh-flow/Services/GithubWebHookProcessor.cs @@ -211,7 +211,7 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor Org = org, Repo = repo, Number = (int)issueNumber, - Content = readme + Content = string.IsNullOrEmpty(readme)? "Sorry, something went wrong": readme }); } else if (skillName == nameof(DevLead) && functionName == nameof(DevLead.Plan)) @@ -223,19 +223,20 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor Org = org, Repo = repo, Number = (int)issueNumber, - Content = plan + Content = string.IsNullOrEmpty(plan)? "Sorry, something went wrong":plan }); } else if (skillName == nameof(Developer) && functionName == nameof(Developer.Implement)) { var dev = _grains.GetGrain(issueNumber, suffix); var code = await dev.GenerateCode(input); + await _ghService.PostComment(new PostCommentRequest { Org = org, Repo = repo, Number = (int)issueNumber, - Content = code + Content = string.IsNullOrEmpty(code)? "Sorry, something went wrong":code }); } else { }// something went wrong diff --git a/src/libs/Microsoft.AI.DevTeam/Actors/Conductor/Conductor.cs b/src/libs/Microsoft.AI.DevTeam/Actors/Conductor/Conductor.cs index 664e6b3ab..51c05b88d 100644 --- a/src/libs/Microsoft.AI.DevTeam/Actors/Conductor/Conductor.cs +++ b/src/libs/Microsoft.AI.DevTeam/Actors/Conductor/Conductor.cs @@ -37,8 +37,6 @@ public class Conductor : Grain, IOrchestrateWorkflows ParentNumber = parentNumber }); var suffix = $"{org}-{repo}"; - var pm = GrainFactory.GetGrain(pmIssue.IssueNumber, suffix); - var devLead = GrainFactory.GetGrain(devLeadIssue.IssueNumber, suffix); var lookup = GrainFactory.GetGrain(suffix); var metadataList = new List{ @@ -81,9 +79,4 @@ public class Conductor : Grain, IOrchestrateWorkflows } await lookup.StoreMetadata(metadataList); } - - public Task ScheduleCommitSandboxRun(CommitRequest commitRequest, MarkTaskCompleteRequest markTaskCompleteRequest) - { - throw new NotImplementedException(); - } } diff --git a/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/DeveloperLead.cs b/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/DeveloperLead.cs index a86f18ef7..46945c8ff 100644 --- a/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/DeveloperLead.cs +++ b/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/DeveloperLead.cs @@ -1,4 +1,5 @@ using Microsoft.AI.DevTeam.Skills; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Orchestration; @@ -9,44 +10,49 @@ namespace Microsoft.AI.DevTeam; public class DeveloperLead : SemanticPersona, ILeadDevelopment { private readonly IKernel _kernel; + private readonly ILogger _logger; + protected override string MemorySegment => "dev-lead-memory"; - public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState state) : base(state) + public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState state, ILogger logger) : base(state) { _kernel = kernel; + _logger = logger; } public async Task CreatePlan(string ask) { - // var architectId = Guid.NewGuid(); - // var plan = "this is my plan"; - // var architect = GrainFactory.GetGrain(architectId); - // var review = architect.ReviewPlan(plan); - // return Task.FromResult(plan); - - var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 }); - var context = new ContextVariables(); - context.Set("input", ask); - if (_state.State.History == null) _state.State.History = new List(); - _state.State.History.Add(new ChatHistoryItem + try { - Message = ask, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.User - }); - await AddWafContext(_kernel, ask, context); - context.Set("input", ask); + var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 }); + var context = new ContextVariables(); + context.Set("input", ask); + if (_state.State.History == null) _state.State.History = new List(); + _state.State.History.Add(new ChatHistoryItem + { + Message = ask, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.User + }); + await AddWafContext(_kernel, ask, context); + context.Set("input", ask); - var result = await _kernel.RunAsync(context, function); - var resultMessage = result.ToString(); - _state.State.History.Add(new ChatHistoryItem + var result = await _kernel.RunAsync(context, function); + var resultMessage = result.ToString(); + _state.State.History.Add(new ChatHistoryItem + { + Message = resultMessage, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.Agent + }); + await _state.WriteStateAsync(); + + return resultMessage; + } + catch (Exception ex) { - Message = resultMessage, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.Agent - }); - await _state.WriteStateAsync(); - - return resultMessage; + _logger.LogError(ex, "Error creating development plan"); + return default; + } } public Task GetLatestPlan() @@ -55,11 +61,6 @@ public class DeveloperLead : SemanticPersona, ILeadDevelopment var response = JsonSerializer.Deserialize(plan); return Task.FromResult(response); } - - public Task BuildUnderstanding(string content) - { - throw new NotImplementedException(); - } } [GenerateSerializer] diff --git a/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/ILeadDevelopment.cs b/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/ILeadDevelopment.cs index 8c479f7e0..6799956c0 100644 --- a/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/ILeadDevelopment.cs +++ b/src/libs/Microsoft.AI.DevTeam/Actors/DevLead/ILeadDevelopment.cs @@ -1,6 +1,6 @@ namespace Microsoft.AI.DevTeam; -public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory, IUnderstand +public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory { Task CreatePlan(string ask); Task GetLatestPlan(); diff --git a/src/libs/Microsoft.AI.DevTeam/Actors/Developer/Developer.cs b/src/libs/Microsoft.AI.DevTeam/Actors/Developer/Developer.cs index 0b4e01760..f9826d281 100644 --- a/src/libs/Microsoft.AI.DevTeam/Actors/Developer/Developer.cs +++ b/src/libs/Microsoft.AI.DevTeam/Actors/Developer/Developer.cs @@ -1,4 +1,5 @@ using Microsoft.AI.DevTeam.Skills; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Orchestration; @@ -9,41 +10,52 @@ namespace Microsoft.AI.DevTeam; public class Dev : SemanticPersona, IDevelopCode { private readonly IKernel _kernel; - protected override string MemorySegment => "dev-memory"; + private readonly ILogger _logger; - public Dev(IKernel kernel, [PersistentState("state", "messages")]IPersistentState state) : base(state) + protected override string MemorySegment => "dev-memory"; + + public Dev(IKernel kernel, [PersistentState("state", "messages")] IPersistentState state, ILogger logger) : base(state) { _kernel = kernel; + _logger = logger; } public async Task GenerateCode(string ask) { - var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 }); - var context = new ContextVariables(); - if (_state.State.History == null) _state.State.History = new List(); - _state.State.History.Add(new ChatHistoryItem + try { - Message = ask, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.User - }); - await AddWafContext(_kernel, ask, context); - context.Set("input", ask); + var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 }); + var context = new ContextVariables(); + if (_state.State.History == null) _state.State.History = new List(); + _state.State.History.Add(new ChatHistoryItem + { + Message = ask, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.User + }); + await AddWafContext(_kernel, ask, context); + context.Set("input", ask); - var result = await _kernel.RunAsync(context, function); - var resultMessage = result.ToString(); - _state.State.History.Add(new ChatHistoryItem + var result = await _kernel.RunAsync(context, function); + var resultMessage = result.ToString(); + _state.State.History.Add(new ChatHistoryItem + { + Message = resultMessage, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.Agent + }); + await _state.WriteStateAsync(); + + return resultMessage; + } + catch(Exception ex) { - Message = resultMessage, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.Agent - }); - await _state.WriteStateAsync(); - - return resultMessage; + _logger.LogError(ex, "Error generating code"); + return default; + } } - + public Task ReviewPlan(string plan) { diff --git a/src/libs/Microsoft.AI.DevTeam/Actors/ProductManager/ProductManager.cs b/src/libs/Microsoft.AI.DevTeam/Actors/ProductManager/ProductManager.cs index bb93e1e80..055b4815e 100644 --- a/src/libs/Microsoft.AI.DevTeam/Actors/ProductManager/ProductManager.cs +++ b/src/libs/Microsoft.AI.DevTeam/Actors/ProductManager/ProductManager.cs @@ -1,4 +1,5 @@ using Microsoft.AI.DevTeam.Skills; +using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Orchestration; @@ -8,36 +9,47 @@ namespace Microsoft.AI.DevTeam; public class ProductManager : SemanticPersona, IManageProduct { private readonly IKernel _kernel; + private readonly ILogger _logger; + protected override string MemorySegment => "pm-memory"; - public ProductManager(IKernel kernel,[PersistentState("state", "messages")] IPersistentState state) : base(state) + public ProductManager(IKernel kernel, [PersistentState("state", "messages")] IPersistentState state, ILogger logger) : base(state) { _kernel = kernel; + _logger = logger; } public async Task CreateReadme(string ask) { - var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 }); - var context = new ContextVariables(); - context.Set("input", ask); - if(_state.State.History == null) _state.State.History = new List(); - _state.State.History.Add(new ChatHistoryItem + try { - Message = ask, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.User - }); - await AddWafContext(_kernel, ask, context); - context.Set("input", ask); + var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 }); + var context = new ContextVariables(); + context.Set("input", ask); + if (_state.State.History == null) _state.State.History = new List(); + _state.State.History.Add(new ChatHistoryItem + { + Message = ask, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.User + }); + await AddWafContext(_kernel, ask, context); + context.Set("input", ask); - var result = await _kernel.RunAsync(context, function); - var resultMessage = result.ToString(); - _state.State.History.Add(new ChatHistoryItem + var result = await _kernel.RunAsync(context, function); + var resultMessage = result.ToString(); + _state.State.History.Add(new ChatHistoryItem + { + Message = resultMessage, + Order = _state.State.History.Count + 1, + UserType = ChatUserType.Agent + }); + await _state.WriteStateAsync(); + return resultMessage; + } + catch (Exception ex) { - Message = resultMessage, - Order = _state.State.History.Count + 1, - UserType = ChatUserType.Agent - }); - await _state.WriteStateAsync(); - return resultMessage; + _logger.LogError(ex, "Error creating readme"); + return default; + } } } diff --git a/src/libs/Microsoft.AI.DevTeam/Services/AzureService.cs b/src/libs/Microsoft.AI.DevTeam/Services/AzureService.cs index 5326c285e..17229fea7 100644 --- a/src/libs/Microsoft.AI.DevTeam/Services/AzureService.cs +++ b/src/libs/Microsoft.AI.DevTeam/Services/AzureService.cs @@ -7,6 +7,7 @@ using Azure.ResourceManager.ContainerInstance; using Azure.ResourceManager.ContainerInstance.Models; using Azure.ResourceManager.Resources; using Azure.Storage.Files.Shares; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -15,108 +16,142 @@ namespace Microsoft.AI.DevTeam; public class AzureService : IManageAzure { private readonly AzureOptions _azSettings; + private readonly ILogger _logger; - public AzureService(IOptions azOptions) + public AzureService(IOptions azOptions, ILogger logger) { _azSettings = azOptions.Value; + _logger = logger; } public async Task DeleteSandbox(string sandboxId) { - var client = new ArmClient(new DefaultAzureCredential()); - var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); - var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); + try + { + var client = new ArmClient(new DefaultAzureCredential()); + var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); + var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); + + var collection = resourceGroupResource.GetContainerGroups(); + var containerGroup = await collection.GetAsync(sandboxId); + await containerGroup.Value.DeleteAsync(WaitUntil.Started); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting sandbox"); + } - var collection = resourceGroupResource.GetContainerGroups(); - var containerGroup = await collection.GetAsync(sandboxId); - await containerGroup.Value.DeleteAsync(WaitUntil.Started); } public async Task IsSandboxCompleted(string sandboxId) { - var client = new ArmClient(new DefaultAzureCredential()); - var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); - var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); + try + { + var client = new ArmClient(new DefaultAzureCredential()); + var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); + var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); - var collection = resourceGroupResource.GetContainerGroups(); - var containerGroup = await collection.GetAsync(sandboxId); - return containerGroup.Value.Data.ProvisioningState == "Succeeded" - && containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated"; + var collection = resourceGroupResource.GetContainerGroups(); + var containerGroup = await collection.GetAsync(sandboxId); + return containerGroup.Value.Data.ProvisioningState == "Succeeded" + && containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated"; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking sandbox status"); + return false; + } } public async Task RunInSandbox(SandboxRequest request) { - var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ? + try + { + var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ? new ArmClient(new AzureCliCredential()) : new ArmClient(new ManagedIdentityCredential(_azSettings.ManagedIdentity)); - var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}"; - var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); - var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); - var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh"; - var collection = resourceGroupResource.GetContainerGroups(); - var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[] - { - new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1))) - { - Command = { "/bin/bash", $"{scriptPath}" }, - VolumeMounts = + var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}"; + var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup); + var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId); + var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh"; + var collection = resourceGroupResource.GetContainerGroups(); + var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[] + { + new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1))) { - new ContainerVolumeMount("azfiles","/azfiles/") + Command = { "/bin/bash", $"{scriptPath}" }, + VolumeMounts = { - IsReadOnly = false, - } - }, - }}, ContainerInstanceOperatingSystemType.Linux) - { - Volumes = - { - new ContainerVolume("azfiles") + new ContainerVolumeMount("azfiles","/azfiles/") + { + IsReadOnly = false, + } + }, + }}, ContainerInstanceOperatingSystemType.Linux) + { + Volumes = { - AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName) + new ContainerVolume("azfiles") { - StorageAccountKey = _azSettings.FilesAccountKey + AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName) + { + StorageAccountKey = _azSettings.FilesAccountKey + }, }, }, - }, - RestartPolicy = ContainerGroupRestartPolicy.Never, - Sku = ContainerGroupSku.Standard, - Priority = ContainerGroupPriority.Regular - }; - await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data); + RestartPolicy = ContainerGroupRestartPolicy.Never, + Sku = ContainerGroupSku.Standard, + Priority = ContainerGroupPriority.Regular + }; + await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error running sandbox"); + } + } public async Task Store(SaveOutputRequest request) { - var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; - var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}"; - - var fileName = $"{request.FileName}.{request.Extension}"; - - var share = new ShareClient(connectionString, _azSettings.FilesShareName); - await share.CreateIfNotExistsAsync(); - await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ; - - var parentDir = share.GetDirectoryClient(parentDirName); - await parentDir.CreateIfNotExistsAsync(); - - var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}"); - await parentIssueDir.CreateIfNotExistsAsync(); - - var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}"); - await directory.CreateIfNotExistsAsync(); - - var file = directory.GetFileClient(fileName); - // hack to enable script to save files in the same directory - var cwdHack = "#!/bin/bash\n cd $(dirname $0)"; - var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output; - using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output))) + try { - await file.CreateAsync(stream.Length); - await file.UploadRangeAsync( - new HttpRange(0, stream.Length), - stream); + var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; + var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}"; + + var fileName = $"{request.FileName}.{request.Extension}"; + + var share = new ShareClient(connectionString, _azSettings.FilesShareName); + await share.CreateIfNotExistsAsync(); + await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ; + + var parentDir = share.GetDirectoryClient(parentDirName); + await parentDir.CreateIfNotExistsAsync(); + + var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}"); + await parentIssueDir.CreateIfNotExistsAsync(); + + var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}"); + await directory.CreateIfNotExistsAsync(); + + var file = directory.GetFileClient(fileName); + // hack to enable script to save files in the same directory + var cwdHack = "#!/bin/bash\n cd $(dirname $0)"; + var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output; + using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output))) + { + await file.CreateAsync(stream.Length); + await file.UploadRangeAsync( + new HttpRange(0, stream.Length), + stream); + } } + catch (Exception ex) + { + _logger.LogError(ex, "Error storing output"); + } + } } diff --git a/src/libs/Microsoft.AI.DevTeam/Services/CodeAnalyzer.cs b/src/libs/Microsoft.AI.DevTeam/Services/CodeAnalyzer.cs index 7153805ed..3e8cb3371 100644 --- a/src/libs/Microsoft.AI.DevTeam/Services/CodeAnalyzer.cs +++ b/src/libs/Microsoft.AI.DevTeam/Services/CodeAnalyzer.cs @@ -1,6 +1,7 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Octokit.Internal; @@ -14,22 +15,32 @@ public class CodeAnalyzer : IAnalyzeCode { private readonly ServiceOptions _serviceOptions; private readonly HttpClient _httpClient; + private readonly ILogger _logger; - public CodeAnalyzer(IOptions serviceOptions, HttpClient httpClient) + public CodeAnalyzer(IOptions serviceOptions, HttpClient httpClient, ILogger logger) { _serviceOptions = serviceOptions.Value; _httpClient = httpClient; - _httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl); + _logger = logger; + _httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl); } public async Task> Analyze(string content) { - var request = new CodeAnalysisRequest { Content = content }; - var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json"); - var response = await _httpClient.PostAsync("api/AnalyzeCode", body); - var stringResult = await response.Content.ReadAsStringAsync(); - var result = JsonSerializer.Deserialize>(stringResult); - return result; + try + { + var request = new CodeAnalysisRequest { Content = content }; + var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json"); + var response = await _httpClient.PostAsync("api/AnalyzeCode", body); + var stringResult = await response.Content.ReadAsStringAsync(); + var result = JsonSerializer.Deserialize>(stringResult); + return result; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error analyzing code"); + return Enumerable.Empty(); + } } } diff --git a/src/libs/Microsoft.AI.DevTeam/Services/GithubAuthService.cs b/src/libs/Microsoft.AI.DevTeam/Services/GithubAuthService.cs index 8752dcb97..7a5156a77 100644 --- a/src/libs/Microsoft.AI.DevTeam/Services/GithubAuthService.cs +++ b/src/libs/Microsoft.AI.DevTeam/Services/GithubAuthService.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Octokit; @@ -5,32 +6,42 @@ namespace Microsoft.AI.DevTeam; public class GithubAuthService { private readonly GithubOptions _githubSettings; + private readonly ILogger _logger; - public GithubAuthService(IOptions ghOptions) + public GithubAuthService(IOptions ghOptions, ILogger logger) { _githubSettings = ghOptions.Value; + _logger = logger; } public async Task GetGitHubClient() { - // Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file - var generator = new GitHubJwt.GitHubJwtFactory( - new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey), - new GitHubJwt.GitHubJwtFactoryOptions - { - AppIntegrationId = _githubSettings.AppId, // The GitHub App Id - ExpirationSeconds = 600 // 10 minutes is the maximum time allowed - } - ); + try + { + // Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file + var generator = new GitHubJwt.GitHubJwtFactory( + new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey), + new GitHubJwt.GitHubJwtFactoryOptions + { + AppIntegrationId = _githubSettings.AppId, // The GitHub App Id + ExpirationSeconds = 600 // 10 minutes is the maximum time allowed + } + ); - var jwtToken = generator.CreateEncodedJwtToken(); - var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP")) + var jwtToken = generator.CreateEncodedJwtToken(); + var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP")) + { + Credentials = new Credentials(jwtToken, AuthenticationType.Bearer) + }; + var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId); + return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}")) + { + Credentials = new Credentials(response.Token) + }; + } + catch (Exception ex) { - Credentials = new Credentials(jwtToken, AuthenticationType.Bearer) - }; - var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId); - return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}")) - { - Credentials = new Credentials(response.Token) - }; + _logger.LogError(ex, "Error getting GitHub client"); + return default; + } } } \ No newline at end of file diff --git a/src/libs/Microsoft.AI.DevTeam/Services/GithubService.cs b/src/libs/Microsoft.AI.DevTeam/Services/GithubService.cs index f480b1f3b..3fd7d1e6a 100644 --- a/src/libs/Microsoft.AI.DevTeam/Services/GithubService.cs +++ b/src/libs/Microsoft.AI.DevTeam/Services/GithubService.cs @@ -25,126 +25,195 @@ public class GithubService : IManageGithub public async Task CommitToBranch(CommitRequest request) { - var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; - - var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}"; - var share = new ShareClient(connectionString, _azSettings.FilesShareName); - var directory = share.GetDirectoryClient(dirName); - - var remaining = new Queue(); - remaining.Enqueue(directory); - while (remaining.Count > 0) + try { - var dir = remaining.Dequeue(); - await foreach (var item in dir.GetFilesAndDirectoriesAsync()) - { - if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR - { - try - { - var file = dir.GetFileClient(item.Name); - var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "") - .Replace($"{dirName}/", ""); - var fileStream = await file.OpenReadAsync(); - using (var reader = new StreamReader(fileStream, Encoding.UTF8)) - { - var value = reader.ReadToEnd(); + var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net"; - await _ghClient.Repository.Content.CreateFile( - request.Org, request.Repo, filePath, - new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message + var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}"; + var share = new ShareClient(connectionString, _azSettings.FilesShareName); + var directory = share.GetDirectoryClient(dirName); + + var remaining = new Queue(); + remaining.Enqueue(directory); + while (remaining.Count > 0) + { + var dir = remaining.Dequeue(); + await foreach (var item in dir.GetFilesAndDirectoriesAsync()) + { + if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR + { + try + { + var file = dir.GetFileClient(item.Name); + var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "") + .Replace($"{dirName}/", ""); + var fileStream = await file.OpenReadAsync(); + using (var reader = new StreamReader(fileStream, Encoding.UTF8)) + { + var value = reader.ReadToEnd(); + + await _ghClient.Repository.Content.CreateFile( + request.Org, request.Repo, filePath, + new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error while uploading file {item.Name}"); } } - catch (Exception ex) + else if (item.IsDirectory) { - _logger.LogError(ex, $"Error while uploading file {item.Name}"); + remaining.Enqueue(dir.GetSubdirectoryClient(item.Name)); } } - else if (item.IsDirectory) - { - remaining.Enqueue(dir.GetSubdirectoryClient(item.Name)); - } } } + catch (Exception ex) + { + _logger.LogError(ex, "Error committing to branch"); + } } public async Task CreateBranch(CreateBranchRequest request) { - var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); - await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch); + try + { + var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); + await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error creating branch"); + } } public async Task GetMainLanguage(string org, string repo) { - var languages = await _ghClient.Repository.GetAllLanguages(org, repo); - var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First(); - return mainLanguage.Name; + try + { + var languages = await _ghClient.Repository.GetAllLanguages(org, repo); + var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First(); + return mainLanguage.Name; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting main language"); + return default; + } } public async Task CreateIssue(CreateIssueRequest request) { - var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}") + try { - Body = request.Input, + var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}") + { + Body = request.Input, - }; - newIssue.Labels.Add(request.Label); - var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue); - var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}"; - var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody); - return new NewIssueResponse + }; + newIssue.Labels.Add(request.Label); + var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue); + var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}"; + var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody); + return new NewIssueResponse + { + IssueNumber = issue.Number, + CommentId = comment.Id + }; + } + catch (Exception ex) { - IssueNumber = issue.Number, - CommentId = comment.Id - }; - + _logger.LogError(ex, "Error creating issue"); + return default; + } } public async Task CreatePR(CreatePRRequest request) { - var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); - await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch)); + try + { + var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo); + await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch)); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error creating PR"); + } + } public async Task MarkTaskComplete(MarkTaskCompleteRequest request) { - var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId); - var updatedComment = comment.Body.Replace("[ ]", "[x]"); - await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment); + try + { + var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId); + var updatedComment = comment.Body.Replace("[ ]", "[x]"); + await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error marking task complete"); + } + } public async Task PostComment(PostCommentRequest request) { - await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content); - } - - public async Task> GetFiles(string org, string repo, string branch, Func filter) - { - var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch); - return await CollectFiles(org, repo, branch, items, filter); - } - - private async Task> CollectFiles(string org, string repo, string branch, IReadOnlyList items, Func filter) - { - var result = new List(); - foreach(var item in items) + try { - if (item.Type == ContentType.File && filter(item)) - { - var content = await _httpClient.GetStringAsync(item.DownloadUrl); - result.Add(new FileResponse - { - Name = item.Name, - Content = content - }); - } - else if (item.Type == ContentType.Dir) - { - var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo,item.Path, branch); - result.AddRange(await CollectFiles(org, repo, branch, subItems, filter)); - } + await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error posting comment"); + } + + } + + public async Task> GetFiles(string org, string repo, string branch, Func filter) + { + try + { + var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch); + return await CollectFiles(org, repo, branch, items, filter); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error getting files"); + return Enumerable.Empty(); + } + } + + private async Task> CollectFiles(string org, string repo, string branch, IReadOnlyList items, Func filter) + { + try + { + var result = new List(); + foreach (var item in items) + { + if (item.Type == ContentType.File && filter(item)) + { + var content = await _httpClient.GetStringAsync(item.DownloadUrl); + result.Add(new FileResponse + { + Name = item.Name, + Content = content + }); + } + else if (item.Type == ContentType.Dir) + { + var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, item.Path, branch); + result.AddRange(await CollectFiles(org, repo, branch, subItems, filter)); + } + } + return result; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error collecting files"); + return Enumerable.Empty(); } - return result; } } @@ -164,7 +233,7 @@ public interface IManageGithub Task CommitToBranch(CommitRequest request); Task PostComment(PostCommentRequest request); - Task> GetFiles(string org, string repo, string branch, Func filter); + Task> GetFiles(string org, string repo, string branch, Func filter); Task GetMainLanguage(string org, string repo); }