Add missing config entries and better config logging (Azure#9062)

* Add missing config entries and better config logging
* Add additional logging to github client factory
* Add better rate limit logging

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureKeyVaultSettings.cs

@@ -39,7 +39,7 @@ public void PostConfigure(string name, T options)
                 {
                     if (DateTimeOffset.UtcNow < cacheEntry.ExpirationTime)
                     {
-                        this.logger.LogInformation("Replacing setting property {PropertyName} with value from cache", property.Name);
+                        this.logger.LogInformation("Replacing setting {PropertyName} with value from cache", property.Name);
                         property.SetValue(options, cacheEntry.Value);
                         continue;
                     }
@@ -52,16 +52,19 @@ public void PostConfigure(string name, T options)
                     var vaultUrl = match.Groups["vault"].Value;
                     var secretName = match.Groups["secret"].Value;
 
+                    this.logger.LogInformation("Setting {PropertyName} points to Key Vault secret url {SecretUrl}", property.Name, propertyValue);
                     try
                     {
                         var secretClient = this.secretClientProvider.GetSecretClient(new Uri(vaultUrl));
-                        this.logger.LogInformation("Replacing setting property {PropertyName} with value from secret {SecretUrl}", property.Name, propertyValue);
 
+                        this.logger.LogInformation("Getting secret value from {SecretUrl}", propertyValue);
                         var response = secretClient.GetSecret(secretName);
                         var secret = response.Value;
 
+                        this.logger.LogInformation("Replacing setting {PropertyName} with value from secret", property.Name);
                         property.SetValue(options, secret.Value);
 
+                        this.logger.LogInformation("Caching secret value for setting {PropertyName}", property.Name);
                         this.valueCache[propertyValue] = (ExpirationTime: DateTimeOffset.UtcNow.AddMinutes(5), secret.Value);
                     }
                     catch (Exception exception)

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureSettings.cs

@@ -36,6 +36,8 @@ public void PostConfigure(string name, PipelineWitnessSettings options)
                     .ConfigureAwait(true)
                     .GetAwaiter()
                     .GetResult();
+
+                this.logger.LogInformation("Loaded {Count} repositories from {Source}", options.GitHubRepositories.Length, options.GitHubRepositoriesSource);
             }
             catch (Exception ex)
             {

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubClientFactory.cs

@@ -8,6 +8,7 @@
 using System.Threading.Tasks;
 using Azure.Sdk.Tools.PipelineWitness.Configuration;
 using Azure.Sdk.Tools.PipelineWitness.Utilities;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.IdentityModel.JsonWebTokens;
 using Microsoft.IdentityModel.Tokens;
@@ -20,23 +21,27 @@ public class GitHubClientFactory
     private readonly TimeSpan processTimeout = TimeSpan.FromSeconds(13);
     private readonly PipelineWitnessSettings settings;
     private readonly ProductHeaderValue productHeaderValue;
+    private readonly ILogger<GitHubClientFactory> logger;
 
-    public GitHubClientFactory(IOptions<PipelineWitnessSettings> options)
+    public GitHubClientFactory(ILogger<GitHubClientFactory> logger, IOptions<PipelineWitnessSettings> options)
     {
         this.settings = options.Value;
 
         string version = typeof(GitHubClientFactory).Assembly.GetName().Version.ToString();
         this.productHeaderValue = new("PipelineWitness", version);
+        this.logger = logger;
     }
 
     public async Task<IGitHubClient> CreateGitHubClientAsync()
     {
         // If we're running in local dev mode, return a client based on the CLI token
         if (string.IsNullOrEmpty(this.settings.GitHubAppPrivateKey))
         {
+            this.logger.LogDebug("No private key provided, creating cli authenticated client.");
             return await CreateGitHubClientWithCliTokenAsync();
         }
 
+        this.logger.LogDebug("Creating app token authenticated client.");
         return CreateGitHubClientWithAppToken();
     }
 
@@ -45,24 +50,30 @@ public async Task<IGitHubClient> CreateGitHubClientAsync(string owner, string re
         // If we're running in local dev mode, return a client based on the CLI token
         if (string.IsNullOrEmpty(this.settings.GitHubAppPrivateKey))
         {
+            this.logger.LogDebug("No private key provided, creating cli authenticated client.");
             return await CreateGitHubClientWithCliTokenAsync();
         }
 
+        this.logger.LogDebug("Creating app token authenticated client.");
         GitHubClient appClient = CreateGitHubClientWithAppToken();
 
         Installation installation;
 
         try
         {
+            this.logger.LogDebug("Getting app installation for {Owner}/{Repository}.", owner, repository);
             installation = await appClient.GitHubApps.GetRepositoryInstallationForCurrent(owner, repository);
         }
         catch (NotFoundException)
         {
+            this.logger.LogError("The GitHub App is not installed on the repository {Owner}/{Repository}.", owner, repository);
             throw new InvalidOperationException($"The GitHub App is not installed on the repository {owner}/{repository}");
         }
 
+        this.logger.LogDebug("Getting installation token for {Owner}/{Repository}.", owner, repository);
         AccessToken accessToken = await appClient.GitHubApps.CreateInstallationToken(installation.Id);
 
+        this.logger.LogDebug("Creating installation token authenticated client.");
         Credentials installationCredentials = new(accessToken.Token);
 
         GitHubClient installationClient = new(this.productHeaderValue)
@@ -126,6 +137,7 @@ private Credentials CreateAppCredentials()
 
     private async Task<Credentials> GetCliCredentialsAsync()
     {
+        this.logger.LogDebug("Creating GitHub token using gh cli.");
         Process process = new()
         {
             StartInfo = GetGitHubCliProcessStartInfo(),

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/MissingGitHubActionsWorker.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Sdk.Tools.PipelineWitness.Configuration;
@@ -69,7 +70,7 @@ protected override async Task ProcessAsync(CancellationToken cancellationToken)
                 {
                     var client = await this.clientFactory.CreateGitHubClientAsync();
                     var rateLimit = await client.RateLimit.GetRateLimits();
-                    this.logger.LogInformation("Rate limit details: {RateLimit}", rateLimit.Resources);
+                    this.logger.LogInformation("Rate limit details: {RateLimit}", JsonSerializer.Serialize(rateLimit.Resources));
                 }
                 catch (Exception rateLimitException)
                 {

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/RunCompleteQueueWorker.cs

@@ -57,7 +57,7 @@ internal override async Task ProcessMessageAsync(QueueMessage message, Cancellat
                 {
                     var client = await this.githubClientFactory.CreateGitHubClientAsync();
                     var rateLimit = await client.RateLimit.GetRateLimits();
-                    this.logger.LogInformation("Rate limit details: {RateLimit}", rateLimit.Resources);
+                    this.logger.LogInformation("Rate limit details: {RateLimit}", JsonSerializer.Serialize(rateLimit.Resources));
                 }
                 catch (Exception rateLimitException)
                 {

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json

@@ -18,9 +18,7 @@
     "BlobStorageAccountUri": "https://pipelinelogstest.blob.core.windows.net",
     "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com",
     "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret",
-    "GitHubAccessToken": null,
-
-    "GitHubRepositoriesSource": "https://raw.githubusercontent.com/Azure/azure-sdk-tools/users/pahallis/missing-build/tools/pipeline-witness/monitored-repos.json",
+    "GitHubAppPrivateKey": null,
 
     "BuildCompleteWorkerCount": 1,
     "GitHubActionRunsWorkerCount": 1,

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json

@@ -53,7 +53,8 @@
 
     "GitHubRepositoriesSource": "https://raw.githubusercontent.com/Azure/azure-sdk-tools/main/tools/pipeline-witness/monitored-repos.json",
     "GitHubWebhookSecret": "https://pipelinewitnessprod.vault.azure.net/secrets/github-webhook-validation-secret",
-    "GitHubAccessToken": "https://pipelinewitnessprod.vault.azure.net/secrets/azuresdk-github-pat",
+    "GitHubAppClientId": "Iv23liEFqcMNX1zoVJDL",
+    "GitHubAppPrivateKey": "https://pipelinewitnessprod.vault.azure.net/secrets/github-app-key",
     "MessageLeasePeriod": "00:03:00",
     "MessageErrorSleepPeriod": "00:00:10",
     "MaxDequeueCount": 5,

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.staging.json

@@ -4,6 +4,6 @@
     "BlobStorageAccountUri": "https://pipelinelogsstaging.blob.core.windows.net",
     "CosmosAccountUri": "https://pipelinewitnessstaging.documents.azure.com",
     "GitHubWebhookSecret": "https://pipelinewitnessstaging.vault.azure.net/secrets/github-webhook-validation-secret",
-    "GitHubAccessToken": "https://pipelinewitnessstaging.vault.azure.net/secrets/azuresdk-github-pat"
+    "GitHubAppPrivateKey": "https://pipelinewitnessstaging.vault.azure.net/secrets/github-app-key"
   }
 }

tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.test.json

@@ -4,6 +4,6 @@
     "BlobStorageAccountUri": "https://pipelinelogstest.blob.core.windows.net",
     "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com",
     "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret",
-    "GitHubAccessToken": "https://pipelinewitnesstest.vault.azure.net/secrets/azuresdk-github-pat"
+    "GitHubAppPrivateKey": "https://pipelinewitnesstest.vault.azure.net/secrets/github-app-key"
   }
 }