add error handling/logging

This commit is contained in:
Kosta Petan 2023-11-09 16:03:38 +00:00
parent d18296f800
commit 45d9282ab2
11 changed files with 420 additions and 269 deletions

View File

@ -5,6 +5,7 @@ using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
using Microsoft.SemanticKernel.Connectors.Memory.Qdrant;
using Microsoft.SemanticKernel.Memory;
using Microsoft.SemanticKernel.Reliability.Basic;
using Octokit.Webhooks;
using Octokit.Webhooks.AspNetCore;
using Orleans.Configuration;
@ -17,7 +18,8 @@ builder.Services.AddHttpClient();
builder.Services.AddSingleton(s =>
{
var ghOptions = s.GetService<IOptions<GithubOptions>>();
var ghService = new GithubAuthService(ghOptions);
var logger = s.GetService<ILogger<GithubAuthService>>();
var ghService = new GithubAuthService(ghOptions, logger);
var client = ghService.GetGitHubClient().Result;
return client;
});
@ -160,5 +162,9 @@ static IKernel CreateKernel(IServiceProvider provider)
return new KernelBuilder()
.WithLoggerFactory(loggerFactory)
.WithAzureChatCompletionService(openAiConfig.DeploymentOrModelId, openAiConfig.Endpoint, openAiConfig.ApiKey, true, openAiConfig.ServiceId, true)
.WithRetryBasic(new BasicRetryConfig {
MaxRetryCount = 5,
UseExponentialBackoff = true
})
.WithMemory(semanticTextMemory).Build();
}

View File

@ -211,7 +211,7 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor
Org = org,
Repo = repo,
Number = (int)issueNumber,
Content = readme
Content = string.IsNullOrEmpty(readme)? "Sorry, something went wrong": readme
});
}
else if (skillName == nameof(DevLead) && functionName == nameof(DevLead.Plan))
@ -223,19 +223,20 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor
Org = org,
Repo = repo,
Number = (int)issueNumber,
Content = plan
Content = string.IsNullOrEmpty(plan)? "Sorry, something went wrong":plan
});
}
else if (skillName == nameof(Developer) && functionName == nameof(Developer.Implement))
{
var dev = _grains.GetGrain<IDevelopCode>(issueNumber, suffix);
var code = await dev.GenerateCode(input);
await _ghService.PostComment(new PostCommentRequest
{
Org = org,
Repo = repo,
Number = (int)issueNumber,
Content = code
Content = string.IsNullOrEmpty(code)? "Sorry, something went wrong":code
});
}
else { }// something went wrong

View File

@ -37,8 +37,6 @@ public class Conductor : Grain, IOrchestrateWorkflows
ParentNumber = parentNumber
});
var suffix = $"{org}-{repo}";
var pm = GrainFactory.GetGrain<IManageProduct>(pmIssue.IssueNumber, suffix);
var devLead = GrainFactory.GetGrain<ILeadDevelopment>(devLeadIssue.IssueNumber, suffix);
var lookup = GrainFactory.GetGrain<ILookupMetadata>(suffix);
var metadataList = new List<StoreMetadataPairs>{
@ -81,9 +79,4 @@ public class Conductor : Grain, IOrchestrateWorkflows
}
await lookup.StoreMetadata(metadataList);
}
public Task ScheduleCommitSandboxRun(CommitRequest commitRequest, MarkTaskCompleteRequest markTaskCompleteRequest)
{
throw new NotImplementedException();
}
}

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration;
@ -9,44 +10,49 @@ namespace Microsoft.AI.DevTeam;
public class DeveloperLead : SemanticPersona, ILeadDevelopment
{
private readonly IKernel _kernel;
private readonly ILogger<DeveloperLead> _logger;
protected override string MemorySegment => "dev-lead-memory";
public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state) : base(state)
public DeveloperLead(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<DeveloperLead> logger) : base(state)
{
_kernel = kernel;
_logger = logger;
}
public async Task<string> CreatePlan(string ask)
{
// var architectId = Guid.NewGuid();
// var plan = "this is my plan";
// var architect = GrainFactory.GetGrain<IArchitectSolutions>(architectId);
// var review = architect.ReviewPlan(plan);
// return Task.FromResult(plan);
var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 });
var context = new ContextVariables();
context.Set("input", ask);
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
try
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var function = _kernel.CreateSemanticFunction(DevLead.Plan, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.4, TopP = 1 });
var context = new ContextVariables();
context.Set("input", ask);
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch (Exception ex)
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
_logger.LogError(ex, "Error creating development plan");
return default;
}
}
public Task<DevLeadPlanResponse> GetLatestPlan()
@ -55,11 +61,6 @@ public class DeveloperLead : SemanticPersona, ILeadDevelopment
var response = JsonSerializer.Deserialize<DevLeadPlanResponse>(plan);
return Task.FromResult(response);
}
public Task<string> BuildUnderstanding(string content)
{
throw new NotImplementedException();
}
}
[GenerateSerializer]

View File

@ -1,6 +1,6 @@
namespace Microsoft.AI.DevTeam;
public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory, IUnderstand
public interface ILeadDevelopment: IGrainWithIntegerCompoundKey, IChatHistory
{
Task<string> CreatePlan(string ask);
Task<DevLeadPlanResponse> GetLatestPlan();

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration;
@ -9,41 +10,52 @@ namespace Microsoft.AI.DevTeam;
public class Dev : SemanticPersona, IDevelopCode
{
private readonly IKernel _kernel;
protected override string MemorySegment => "dev-memory";
private readonly ILogger<Dev> _logger;
public Dev(IKernel kernel, [PersistentState("state", "messages")]IPersistentState<SemanticPersonaState> state) : base(state)
protected override string MemorySegment => "dev-memory";
public Dev(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<Dev> logger) : base(state)
{
_kernel = kernel;
_logger = logger;
}
public async Task<string> GenerateCode(string ask)
{
var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
var context = new ContextVariables();
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
try
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var function = _kernel.CreateSemanticFunction(Developer.Implement, new OpenAIRequestSettings { MaxTokens = 15000, Temperature = 0.8, TopP = 1 });
var context = new ContextVariables();
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch(Exception ex)
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
_logger.LogError(ex, "Error generating code");
return default;
}
}
public Task<string> ReviewPlan(string plan)
{

View File

@ -1,4 +1,5 @@
using Microsoft.AI.DevTeam.Skills;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.AI.OpenAI;
using Microsoft.SemanticKernel.Orchestration;
@ -8,36 +9,47 @@ namespace Microsoft.AI.DevTeam;
public class ProductManager : SemanticPersona, IManageProduct
{
private readonly IKernel _kernel;
private readonly ILogger<ProductManager> _logger;
protected override string MemorySegment => "pm-memory";
public ProductManager(IKernel kernel,[PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state) : base(state)
public ProductManager(IKernel kernel, [PersistentState("state", "messages")] IPersistentState<SemanticPersonaState> state, ILogger<ProductManager> logger) : base(state)
{
_kernel = kernel;
_logger = logger;
}
public async Task<string> CreateReadme(string ask)
{
var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 });
var context = new ContextVariables();
context.Set("input", ask);
if(_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
try
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var function = _kernel.CreateSemanticFunction(PM.Readme, new OpenAIRequestSettings { MaxTokens = 10000, Temperature = 0.6, TopP = 1 });
var context = new ContextVariables();
context.Set("input", ask);
if (_state.State.History == null) _state.State.History = new List<ChatHistoryItem>();
_state.State.History.Add(new ChatHistoryItem
{
Message = ask,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.User
});
await AddWafContext(_kernel, ask, context);
context.Set("input", ask);
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
var result = await _kernel.RunAsync(context, function);
var resultMessage = result.ToString();
_state.State.History.Add(new ChatHistoryItem
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
}
catch (Exception ex)
{
Message = resultMessage,
Order = _state.State.History.Count + 1,
UserType = ChatUserType.Agent
});
await _state.WriteStateAsync();
return resultMessage;
_logger.LogError(ex, "Error creating readme");
return default;
}
}
}

View File

@ -7,6 +7,7 @@ using Azure.ResourceManager.ContainerInstance;
using Azure.ResourceManager.ContainerInstance.Models;
using Azure.ResourceManager.Resources;
using Azure.Storage.Files.Shares;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@ -15,108 +16,142 @@ namespace Microsoft.AI.DevTeam;
public class AzureService : IManageAzure
{
private readonly AzureOptions _azSettings;
private readonly ILogger<AzureService> _logger;
public AzureService(IOptions<AzureOptions> azOptions)
public AzureService(IOptions<AzureOptions> azOptions, ILogger<AzureService> logger)
{
_azSettings = azOptions.Value;
_logger = logger;
}
public async Task DeleteSandbox(string sandboxId)
{
var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
try
{
var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
await containerGroup.Value.DeleteAsync(WaitUntil.Started);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error deleting sandbox");
}
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
await containerGroup.Value.DeleteAsync(WaitUntil.Started);
}
public async Task<bool> IsSandboxCompleted(string sandboxId)
{
var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
try
{
var client = new ArmClient(new DefaultAzureCredential());
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
return containerGroup.Value.Data.ProvisioningState == "Succeeded"
&& containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated";
var collection = resourceGroupResource.GetContainerGroups();
var containerGroup = await collection.GetAsync(sandboxId);
return containerGroup.Value.Data.ProvisioningState == "Succeeded"
&& containerGroup.Value.Data.Containers.First().InstanceView.CurrentState.State == "Terminated";
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking sandbox status");
return false;
}
}
public async Task RunInSandbox(SandboxRequest request)
{
var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ?
try
{
var client = string.IsNullOrEmpty(_azSettings.ManagedIdentity) ?
new ArmClient(new AzureCliCredential())
: new ArmClient(new ManagedIdentityCredential(_azSettings.ManagedIdentity));
var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}";
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh";
var collection = resourceGroupResource.GetContainerGroups();
var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[]
{
new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1)))
{
Command = { "/bin/bash", $"{scriptPath}" },
VolumeMounts =
var runId = $"sk-sandbox-{request.Org}-{request.Repo}-{request.ParentIssueNumber}-{request.IssueNumber}";
var resourceGroupResourceId = ResourceGroupResource.CreateResourceIdentifier(_azSettings.SubscriptionId, _azSettings.ContainerInstancesResourceGroup);
var resourceGroupResource = client.GetResourceGroupResource(resourceGroupResourceId);
var scriptPath = $"/azfiles/output/{request.Org}-{request.Repo}/{request.ParentIssueNumber}/{request.IssueNumber}/run.sh";
var collection = resourceGroupResource.GetContainerGroups();
var data = new ContainerGroupData(new AzureLocation(_azSettings.Location), new ContainerInstanceContainer[]
{
new ContainerInstanceContainer(runId,_azSettings.SandboxImage,new ContainerResourceRequirements(new ContainerResourceRequestsContent(1.5,1)))
{
new ContainerVolumeMount("azfiles","/azfiles/")
Command = { "/bin/bash", $"{scriptPath}" },
VolumeMounts =
{
IsReadOnly = false,
}
},
}}, ContainerInstanceOperatingSystemType.Linux)
{
Volumes =
{
new ContainerVolume("azfiles")
new ContainerVolumeMount("azfiles","/azfiles/")
{
IsReadOnly = false,
}
},
}}, ContainerInstanceOperatingSystemType.Linux)
{
Volumes =
{
AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName)
new ContainerVolume("azfiles")
{
StorageAccountKey = _azSettings.FilesAccountKey
AzureFile = new ContainerInstanceAzureFileVolume(_azSettings.FilesShareName,_azSettings.FilesAccountName)
{
StorageAccountKey = _azSettings.FilesAccountKey
},
},
},
},
RestartPolicy = ContainerGroupRestartPolicy.Never,
Sku = ContainerGroupSku.Standard,
Priority = ContainerGroupPriority.Regular
};
await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data);
RestartPolicy = ContainerGroupRestartPolicy.Never,
Sku = ContainerGroupSku.Standard,
Priority = ContainerGroupPriority.Regular
};
await collection.CreateOrUpdateAsync(WaitUntil.Completed, runId, data);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error running sandbox");
}
}
public async Task Store(SaveOutputRequest request)
{
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}";
var fileName = $"{request.FileName}.{request.Extension}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
await share.CreateIfNotExistsAsync();
await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ;
var parentDir = share.GetDirectoryClient(parentDirName);
await parentDir.CreateIfNotExistsAsync();
var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}");
await parentIssueDir.CreateIfNotExistsAsync();
var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}");
await directory.CreateIfNotExistsAsync();
var file = directory.GetFileClient(fileName);
// hack to enable script to save files in the same directory
var cwdHack = "#!/bin/bash\n cd $(dirname $0)";
var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output)))
try
{
await file.CreateAsync(stream.Length);
await file.UploadRangeAsync(
new HttpRange(0, stream.Length),
stream);
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
var parentDirName = $"{request.Directory}/{request.Org}-{request.Repo}";
var fileName = $"{request.FileName}.{request.Extension}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
await share.CreateIfNotExistsAsync();
await share.GetDirectoryClient($"{request.Directory}").CreateIfNotExistsAsync(); ;
var parentDir = share.GetDirectoryClient(parentDirName);
await parentDir.CreateIfNotExistsAsync();
var parentIssueDir = parentDir.GetSubdirectoryClient($"{request.ParentIssueNumber}");
await parentIssueDir.CreateIfNotExistsAsync();
var directory = parentIssueDir.GetSubdirectoryClient($"{request.IssueNumber}");
await directory.CreateIfNotExistsAsync();
var file = directory.GetFileClient(fileName);
// hack to enable script to save files in the same directory
var cwdHack = "#!/bin/bash\n cd $(dirname $0)";
var output = request.Extension == "sh" ? request.Output.Replace("#!/bin/bash", cwdHack) : request.Output;
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(output)))
{
await file.CreateAsync(stream.Length);
await file.UploadRangeAsync(
new HttpRange(0, stream.Length),
stream);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error storing output");
}
}
}

View File

@ -1,6 +1,7 @@
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Octokit.Internal;
@ -14,22 +15,32 @@ public class CodeAnalyzer : IAnalyzeCode
{
private readonly ServiceOptions _serviceOptions;
private readonly HttpClient _httpClient;
private readonly ILogger<CodeAnalyzer> _logger;
public CodeAnalyzer(IOptions<ServiceOptions> serviceOptions, HttpClient httpClient)
public CodeAnalyzer(IOptions<ServiceOptions> serviceOptions, HttpClient httpClient, ILogger<CodeAnalyzer> logger)
{
_serviceOptions = serviceOptions.Value;
_httpClient = httpClient;
_httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl);
_logger = logger;
_httpClient.BaseAddress = new Uri(_serviceOptions.IngesterUrl);
}
public async Task<IEnumerable<CodeAnalysis>> Analyze(string content)
{
var request = new CodeAnalysisRequest { Content = content };
var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync("api/AnalyzeCode", body);
var stringResult = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<IEnumerable<CodeAnalysis>>(stringResult);
return result;
try
{
var request = new CodeAnalysisRequest { Content = content };
var body = new StringContent(JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
var response = await _httpClient.PostAsync("api/AnalyzeCode", body);
var stringResult = await response.Content.ReadAsStringAsync();
var result = JsonSerializer.Deserialize<IEnumerable<CodeAnalysis>>(stringResult);
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error analyzing code");
return Enumerable.Empty<CodeAnalysis>();
}
}
}

View File

@ -1,3 +1,4 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Octokit;
@ -5,32 +6,42 @@ namespace Microsoft.AI.DevTeam;
public class GithubAuthService
{
private readonly GithubOptions _githubSettings;
private readonly ILogger<GithubAuthService> _logger;
public GithubAuthService(IOptions<GithubOptions> ghOptions)
public GithubAuthService(IOptions<GithubOptions> ghOptions, ILogger<GithubAuthService> logger)
{
_githubSettings = ghOptions.Value;
_logger = logger;
}
public async Task<GitHubClient> GetGitHubClient()
{
// Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file
var generator = new GitHubJwt.GitHubJwtFactory(
new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey),
new GitHubJwt.GitHubJwtFactoryOptions
{
AppIntegrationId = _githubSettings.AppId, // The GitHub App Id
ExpirationSeconds = 600 // 10 minutes is the maximum time allowed
}
);
try
{
// Use GitHubJwt library to create the GitHubApp Jwt Token using our private certificate PEM file
var generator = new GitHubJwt.GitHubJwtFactory(
new GitHubJwt.StringPrivateKeySource(_githubSettings.AppKey),
new GitHubJwt.GitHubJwtFactoryOptions
{
AppIntegrationId = _githubSettings.AppId, // The GitHub App Id
ExpirationSeconds = 600 // 10 minutes is the maximum time allowed
}
);
var jwtToken = generator.CreateEncodedJwtToken();
var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP"))
var jwtToken = generator.CreateEncodedJwtToken();
var appClient = new GitHubClient(new ProductHeaderValue("SK-DEV-APP"))
{
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer)
};
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId);
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{
Credentials = new Credentials(response.Token)
};
}
catch (Exception ex)
{
Credentials = new Credentials(jwtToken, AuthenticationType.Bearer)
};
var response = await appClient.GitHubApps.CreateInstallationToken(_githubSettings.InstallationId);
return new GitHubClient(new ProductHeaderValue($"SK-DEV-APP-Installation{_githubSettings.InstallationId}"))
{
Credentials = new Credentials(response.Token)
};
_logger.LogError(ex, "Error getting GitHub client");
return default;
}
}
}

View File

@ -25,126 +25,195 @@ public class GithubService : IManageGithub
public async Task CommitToBranch(CommitRequest request)
{
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
var directory = share.GetDirectoryClient(dirName);
var remaining = new Queue<ShareDirectoryClient>();
remaining.Enqueue(directory);
while (remaining.Count > 0)
try
{
var dir = remaining.Dequeue();
await foreach (var item in dir.GetFilesAndDirectoriesAsync())
{
if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR
{
try
{
var file = dir.GetFileClient(item.Name);
var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "")
.Replace($"{dirName}/", "");
var fileStream = await file.OpenReadAsync();
using (var reader = new StreamReader(fileStream, Encoding.UTF8))
{
var value = reader.ReadToEnd();
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={_azSettings.FilesAccountName};AccountKey={_azSettings.FilesAccountKey};EndpointSuffix=core.windows.net";
await _ghClient.Repository.Content.CreateFile(
request.Org, request.Repo, filePath,
new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message
var dirName = $"{request.Dir}/{request.Org}-{request.Repo}/{request.ParentNumber}/{request.Number}";
var share = new ShareClient(connectionString, _azSettings.FilesShareName);
var directory = share.GetDirectoryClient(dirName);
var remaining = new Queue<ShareDirectoryClient>();
remaining.Enqueue(directory);
while (remaining.Count > 0)
{
var dir = remaining.Dequeue();
await foreach (var item in dir.GetFilesAndDirectoriesAsync())
{
if (!item.IsDirectory && item.Name != "run.sh") // we don't want the generated script in the PR
{
try
{
var file = dir.GetFileClient(item.Name);
var filePath = file.Path.Replace($"{_azSettings.FilesShareName}/", "")
.Replace($"{dirName}/", "");
var fileStream = await file.OpenReadAsync();
using (var reader = new StreamReader(fileStream, Encoding.UTF8))
{
var value = reader.ReadToEnd();
await _ghClient.Repository.Content.CreateFile(
request.Org, request.Repo, filePath,
new CreateFileRequest($"Commit message", value, request.Branch)); // TODO: add more meaningfull commit message
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error while uploading file {item.Name}");
}
}
catch (Exception ex)
else if (item.IsDirectory)
{
_logger.LogError(ex, $"Error while uploading file {item.Name}");
remaining.Enqueue(dir.GetSubdirectoryClient(item.Name));
}
}
else if (item.IsDirectory)
{
remaining.Enqueue(dir.GetSubdirectoryClient(item.Name));
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error committing to branch");
}
}
public async Task CreateBranch(CreateBranchRequest request)
{
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch);
try
{
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.Git.Reference.CreateBranch(request.Org, request.Repo, request.Branch, ghRepo.DefaultBranch);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error creating branch");
}
}
public async Task<string> GetMainLanguage(string org, string repo)
{
var languages = await _ghClient.Repository.GetAllLanguages(org, repo);
var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First();
return mainLanguage.Name;
try
{
var languages = await _ghClient.Repository.GetAllLanguages(org, repo);
var mainLanguage = languages.OrderByDescending(l => l.NumberOfBytes).First();
return mainLanguage.Name;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting main language");
return default;
}
}
public async Task<NewIssueResponse> CreateIssue(CreateIssueRequest request)
{
var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}")
try
{
Body = request.Input,
var newIssue = new NewIssue($"{request.Label} chain for #{request.ParentNumber}")
{
Body = request.Input,
};
newIssue.Labels.Add(request.Label);
var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue);
var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}";
var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody);
return new NewIssueResponse
};
newIssue.Labels.Add(request.Label);
var issue = await _ghClient.Issue.Create(request.Org, request.Repo, newIssue);
var commentBody = $" - [ ] #{issue.Number} - tracks {request.Label}";
var comment = await _ghClient.Issue.Comment.Create(request.Org, request.Repo, (int)request.ParentNumber, commentBody);
return new NewIssueResponse
{
IssueNumber = issue.Number,
CommentId = comment.Id
};
}
catch (Exception ex)
{
IssueNumber = issue.Number,
CommentId = comment.Id
};
_logger.LogError(ex, "Error creating issue");
return default;
}
}
public async Task CreatePR(CreatePRRequest request)
{
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch));
try
{
var ghRepo = await _ghClient.Repository.Get(request.Org, request.Repo);
await _ghClient.PullRequest.Create(request.Org, request.Repo, new NewPullRequest($"New app #{request.Number}", request.Branch, ghRepo.DefaultBranch));
}
catch (Exception ex)
{
_logger.LogError(ex, "Error creating PR");
}
}
public async Task MarkTaskComplete(MarkTaskCompleteRequest request)
{
var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId);
var updatedComment = comment.Body.Replace("[ ]", "[x]");
await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment);
try
{
var comment = await _ghClient.Issue.Comment.Get(request.Org, request.Repo, request.CommentId);
var updatedComment = comment.Body.Replace("[ ]", "[x]");
await _ghClient.Issue.Comment.Update(request.Org, request.Repo, request.CommentId, updatedComment);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error marking task complete");
}
}
public async Task PostComment(PostCommentRequest request)
{
await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content);
}
public async Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent,bool> filter)
{
var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch);
return await CollectFiles(org, repo, branch, items, filter);
}
private async Task<IEnumerable<FileResponse>> CollectFiles(string org, string repo, string branch, IReadOnlyList<RepositoryContent> items, Func<RepositoryContent,bool> filter)
{
var result = new List<FileResponse>();
foreach(var item in items)
try
{
if (item.Type == ContentType.File && filter(item))
{
var content = await _httpClient.GetStringAsync(item.DownloadUrl);
result.Add(new FileResponse
{
Name = item.Name,
Content = content
});
}
else if (item.Type == ContentType.Dir)
{
var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo,item.Path, branch);
result.AddRange(await CollectFiles(org, repo, branch, subItems, filter));
}
await _ghClient.Issue.Comment.Create(request.Org, request.Repo, request.Number, request.Content);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error posting comment");
}
}
public async Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent, bool> filter)
{
try
{
var items = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, branch);
return await CollectFiles(org, repo, branch, items, filter);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error getting files");
return Enumerable.Empty<FileResponse>();
}
}
private async Task<IEnumerable<FileResponse>> CollectFiles(string org, string repo, string branch, IReadOnlyList<RepositoryContent> items, Func<RepositoryContent, bool> filter)
{
try
{
var result = new List<FileResponse>();
foreach (var item in items)
{
if (item.Type == ContentType.File && filter(item))
{
var content = await _httpClient.GetStringAsync(item.DownloadUrl);
result.Add(new FileResponse
{
Name = item.Name,
Content = content
});
}
else if (item.Type == ContentType.Dir)
{
var subItems = await _ghClient.Repository.Content.GetAllContentsByRef(org, repo, item.Path, branch);
result.AddRange(await CollectFiles(org, repo, branch, subItems, filter));
}
}
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error collecting files");
return Enumerable.Empty<FileResponse>();
}
return result;
}
}
@ -164,7 +233,7 @@ public interface IManageGithub
Task CommitToBranch(CommitRequest request);
Task PostComment(PostCommentRequest request);
Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent,bool> filter);
Task<IEnumerable<FileResponse>> GetFiles(string org, string repo, string branch, Func<RepositoryContent, bool> filter);
Task<string> GetMainLanguage(string org, string repo);
}