basic functionality of replay log working

.gitignore

@@ -3,6 +3,8 @@
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
 
+graphlog.json
+
 # User-specific files
 *.rsuser
 *.suo

src/kibali/AuthZChecker.cs

@@ -41,7 +41,7 @@
 
         public ProtectedResource Lookup(string url)
         {
             var parsedUrl = new Uri(new Uri("https://example.org/"), url, true);
             var segments = parsedUrl.AbsolutePath.Split("/").Skip(1);
             return Find(UrlTree, segments);
         }
@@ -177,7 +177,11 @@
 
                 var openApiResource = new OpenApiProtectedResource(resource.Value);
                 pathItem.AddExtension("x-permissions", openApiResource);
-                tree.Attach(resource.Key, pathItem, "!");
+                try {
+                    tree.Attach(resource.Key.ToLower(), pathItem, "!");
+                } catch (Exception e) {
+                    Console.WriteLine("Failed to add resource to tree: " + resource.Key + " " + e.Message);
+                }
             }
 
             return tree;

src/kibali/PermissionsDocument.cs

@@ -58,6 +58,10 @@ public static PermissionsDocument LoadFromFolder(string documentPath)
             {
                 try
                 {
+                    if(permissionsFile.EndsWith("provisioningInfo.json"))
+                    {
+                        continue;
+                    }
                     using var stream = new FileStream(permissionsFile, FileMode.Open);
                     var doc = Load(stream);
                     mergedPermissions = mergedPermissions.Concat(doc.Permissions).ToDictionary(x => x.Key, x => x.Value);

src/kibaliTool/LogEntry.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace KibaliTool
+{
+        /*  To produce the log file you can use this Kusto query and then Export to JSON
+
+        AggregatorServiceLogEvent 
+        | where env_time > ago(1h)
+        | where tagId == 30746268
+        | where responseStatusCode >= 200 and responseStatusCode < 300 
+        | project correlationId, requestMethod, incomingUri, tokenClaims, responseStatusCode,tagId
+        | limit 10000
+        */
+
+
+        // "TableName":"Results",
+        // "Columns":[  {"ColumnName":"correlationId","DataType":"String"},
+        //              {"ColumnName":"requestMethod","DataType":"String"},
+        //              {"ColumnName":"incomingUri","DataType":"String"},
+        //              {"ColumnName":"tokenClaims","DataType":"String"},
+        //              {"ColumnName":"responseStatusCode","DataType":"Int64"},
+        //              {"ColumnName":"tagId","DataType":"String"}]
+
+    public class LogEntry
+    {
+
+        public string Method;
+        public string Scheme;
+        public string Url;
+        public string Claims;
+
+        private Regex scpRegex = new Regex(@"scp=([^\;]+)", RegexOptions.Compiled);
+        private Regex rolesRegex = new Regex(@"roles=([^\;]+)", RegexOptions.Compiled);
+        private Regex roleRegex = new Regex("\"(.*?)\"", RegexOptions.Compiled);
+
+        public string[] Permissions;
+
+        private void Process() {
+
+            Url = NormalIzeUrl(Url);
+
+
+            // Calculate Scheme
+            Scheme = Claims.Contains("role") ? "Application" : "DelegatedWork";
+            // Calculate Permissions
+
+            string[] permissionList = System.Array.Empty<string>();
+
+            if (Scheme == "DelegatedWork")
+            {
+                Match match = scpRegex.Match(Claims);
+
+                if (match.Success)
+                {
+                    string scpValues = match.Groups[1].Value.Trim();
+                    permissionList = scpValues.Split(' ');
+                }
+            }
+            else
+            {
+                Match match = rolesRegex.Match(Claims);
+
+                if (match.Success)
+                {
+                    string roleValues = match.Groups[1].Value.Trim();
+                    MatchCollection matches = roleRegex.Matches(roleValues);
+                    permissionList = matches.Select(m => m.Groups[1].Value).ToArray();
+                }
+            }
+            Permissions = permissionList;
+        }
+
+        private string NormalIzeUrl(string url)
+        {
+            // Use regeg to tranform the url to use / as a segment separator 
+            url = Regex.Replace(url, @"\(([^)]*)\)", "/$1").ToLower();
+
+            // Remove $value
+            url = Regex.Replace(url, @"\/\$value", string.Empty);
+
+            return url;
+        }
+
+        public static LogEntry Load(JsonElement row) {
+            var logentry = new LogEntry()
+            {
+                Method = row[RequestMethod].GetString(),
+                Url = row[IncomingUri].GetString(),
+                Claims = row[TokenClaims].GetString()
+            };
+            logentry.Process();
+            return logentry;
+        }
+
+        private const int CorrelationId = 0;
+        private const int RequestMethod = 1;
+        private const int IncomingUri = 2;
+
+        private const int TokenClaims = 3;
+        private const int ResponseStatusCode = 4;
+        private const int TagId = 5;
+    }
+}

src/kibaliTool/Program.cs

@@ -53,13 +53,23 @@ static async Task Main(string[] args)
 
             documentCommand.SetHandler(DocumentCommand.Execute, new DocumentCommandBinder());
 
+            Command replayLogCommand = new Command("replay-log") {
+                ReplayLogCommandBinder.LogFileOption,
+                ReplayLogCommandBinder.SourcePermissionsFolderOption,
+                ReplayLogCommandBinder.LenientMatchOption,
+                ReplayLogCommandBinder.CountOption,
+            };
+
+            replayLogCommand.SetHandler(ReplayLogCommand.Execute, new ReplayLogCommandBinder());
+
             var rootCommand = new RootCommand()
             {
                 importCommand,
                 queryCommand,
                 exportCommand,
                 validateCommand,
-                documentCommand
+                documentCommand,
+                replayLogCommand
             };
 
 
@@ -158,4 +168,23 @@ protected override DocumentCommandParameters GetBoundValue(BindingContext bindin
             };
         }
     }
+
+    internal class ReplayLogCommandBinder : BinderBase<ReplayLogCommandParameters> {
+        public static readonly Option<string> LogFileOption = new(new[] { "--logFile", "--lf" }, "Log File");
+        public static readonly Option<string> SourcePermissionsFolderOption = new(new[] { "--sourcePermissionsFolder", "--fo" }, "Permission Folder");
+        public static readonly Option<bool> LenientMatchOption = new(new[] { "--lenient", "--lm" }, "LenientMatch");
+        public static readonly Option<int> CountOption = new(new[] { "--count", "--c" }, "Count");
+
+        protected override ReplayLogCommandParameters GetBoundValue(BindingContext bindingContext)
+        {
+            return new ReplayLogCommandParameters()
+            {
+                LogFile = bindingContext.ParseResult.GetValueForOption(LogFileOption),
+                SourcePermissionsFolder = bindingContext.ParseResult.GetValueForOption(SourcePermissionsFolderOption),
+                LenientMatch = bindingContext.ParseResult.GetValueForOption(LenientMatchOption),
+                Count = bindingContext.ParseResult.GetValueForOption(CountOption),
+            };
+        }
+
+    }
 }

src/kibaliTool/ReplayLogCommand.cs

@@ -0,0 +1,118 @@
+using Kibali;
+using Microsoft.OpenApi.Models;
+using Microsoft.OpenApi.Readers;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+
+namespace KibaliTool
+{
+    internal class ReplayLogCommandParameters
+    {
+        public string SourcePermissionsFolder;
+        public string LogFile;
+        public bool LenientMatch;
+        public int Count = 100;
+    }
+
+    internal class ReplayLogCommand
+    {
+
+
+        public static async Task<int> Execute(ReplayLogCommandParameters replayLogCommandParameters)
+        {
+            var doc = PermissionsDocument.LoadFromFolder(replayLogCommandParameters.SourcePermissionsFolder);
+
+            var authZChecker = new AuthZChecker() { LenientMatch = replayLogCommandParameters.LenientMatch };
+            authZChecker.Load(doc);
+
+            // Read the JSON log file using a streaming API
+            using var logstream = new FileStream(replayLogCommandParameters.LogFile, FileMode.Open);
+
+            IEnumerable<LogEntry> entries = LoadLogEntries(logstream, replayLogCommandParameters.Count).ToList();
+
+            Stopwatch stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            using var writer = new Utf8JsonWriter(Console.OpenStandardOutput(), new JsonWriterOptions() { Indented = true,SkipValidation = true });
+            writer.WriteStartArray();
+
+            int successRequests = 0;
+
+            foreach (var entry in entries)
+            {
+                string failReason = null;
+                Dictionary<string , List<AcceptableClaim>> supportedSchemes = null;
+                List<AcceptableClaim> acceptableClaims = null;
+                List<string> acceptablePermissions = new List<string>();
+                List<string> relevantPermissions = new List<string>();
+                var resource = authZChecker.FindResource(entry.Url);
+
+                if (resource == null)
+                {
+                    failReason = "No matching resource";
+                }
+                if (failReason == null && !resource.SupportedMethods.TryGetValue(entry.Method, out supportedSchemes))
+                {
+                    failReason = "No matching method";
+                }
+
+                if (failReason == null && !supportedSchemes.TryGetValue(entry.Scheme, out acceptableClaims))
+                {
+                    failReason = "No matching scheme"; 
+                }
+
+                if (failReason == null )
+                {
+                    acceptablePermissions = acceptableClaims.Select(c => c.Permission).ToList();
+                    relevantPermissions = entry.Permissions.Where(claim => acceptablePermissions.Contains(claim)).ToList();
+                }
+
+                if (failReason == null && !relevantPermissions.Any())
+                {
+                    failReason = "No matching permissions";
+                }
+                if (failReason != null) {
+                    writer.WriteStartObject();
+                    writer.WriteString("failReason", failReason);
+                    writer.WriteString("url", entry.Url);
+                    writer.WriteString("method", entry.Method);
+                    writer.WriteString("scheme", entry.Scheme);
+                    if (failReason == "No matching permissions") {
+                        writer.WriteString("loggedClaims", String.Join(",", entry.Permissions));
+                        writer.WriteString("requiredClaims", String.Join(",", acceptablePermissions));
+                    }
+                    writer.WriteEndObject();
+                } else {
+                    successRequests++;
+                }
+            }
+            stopwatch.Stop();   
+
+
+            writer.WriteEndArray();
+            await writer.FlushAsync();
+
+            Console.WriteLine($"Elapsed time: {stopwatch.ElapsedMilliseconds} ms");
+            Console.WriteLine($"Total success requests: {successRequests}");
+            return 0;
+
+        }
+
+
+        public static IEnumerable<LogEntry> LoadLogEntries(Stream stream, int count)
+        {
+            var document = JsonDocument.Parse(stream);
+            var root = document.RootElement;
+            var rows = root.GetProperty("Rows");
+            return rows.EnumerateArray().Take(count).Select(row => LogEntry.Load(row));
+        }
+
+    }
+}