feat: paralellize. Fix offset pols not used

performance-tests/Identity.Pool.Creator/PoolsFile/PoolEntry.cs

@@ -37,15 +37,15 @@ public static bool HasMessagesOffsetPool(this IList<PoolEntry> pools)
         return pools.Any(p => p.Alias.EndsWith("0mc"));
     }
 
-    public static long ExpectedNumberOfRelationships(this IList<PoolEntry> pools)
+    public static long ExpectedNumberOfRelationships(this IList<PoolEntry> pools, bool noError = false)
     {
         var appRelationshipsCount = pools.Where(p => p.IsApp()).Sum(p => p.NumberOfRelationships * p.Amount);
         var connectorRelationshipsCount = pools.Where(p => p.IsConnector()).Sum(p => p.NumberOfRelationships * p.Amount);
 
-        if (appRelationshipsCount != connectorRelationshipsCount)
+        if (noError && appRelationshipsCount != connectorRelationshipsCount)
             return -1;
 
-        return appRelationshipsCount;
+        return appRelationshipsCount > connectorRelationshipsCount ? appRelationshipsCount : connectorRelationshipsCount;
     }
 
     public static long ExpectedNumberOfSentMessages(this IList<PoolEntry> pools)

performance-tests/Identity.Pool.Creator/PoolsFile/PoolsOffset.cs

@@ -66,7 +66,7 @@ private PoolsOffset(long messages, long relationships, long sentMessages, long r
         return offset > 0 ? OffsetDirections.App : OffsetDirections.Connector;
     }
 
-    public static void CreateOffsetPools(IList<PoolEntry> pools, PoolsOffset? offset = null)
+    public static List<PoolEntry> CreateOffsetPools(List<PoolEntry> pools, PoolsOffset? offset = null)
     {
         offset ??= CalculatePoolOffsets(pools);
 
@@ -117,6 +117,8 @@ public static void CreateOffsetPools(IList<PoolEntry> pools, PoolsOffset? offset
             pools.Add(messagesOffsetPool1);
             pools.Add(messagesOffsetPool2);
         }
+
+        return pools;
     }
 }
 

performance-tests/Identity.Pool.Creator/PoolsGenerator/GraphPoolsGenerator.cs

@@ -19,7 +19,7 @@ public class GraphPoolsGenerator
     private readonly string _baseAddress;
     private readonly ClientCredentials _clientCredentials;
     private readonly IPrinter _printer;
-    private readonly List<PoolEntry> _pools;
+    private List<PoolEntry> _pools;
 
     public GraphPoolsGenerator(
         string baseAddress,
@@ -38,7 +38,7 @@ public async Task CreatePools()
     {
         CreateInitialNodes();
 
-        PoolsOffset.CreateOffsetPools(_pools);
+        _pools = PoolsOffset.CreateOffsetPools(_pools);
 
         DeepGenerateV2();
 

performance-tests/Identity.Pool.Creator/PoolsGenerator/PoolsGenerator.cs

@@ -1,9 +1,7 @@
 using System.Text;
 using Backbone.ConsumerApi.Sdk;
 using Backbone.ConsumerApi.Sdk.Authentication;
-using Backbone.ConsumerApi.Sdk.Endpoints.Messages.Types.Requests;
 using Backbone.ConsumerApi.Sdk.Endpoints.RelationshipTemplates.Types.Requests;
-using Backbone.Crypto;
 using Backbone.Identity.Pool.Creator.Application.MessageDistributor;
 using Backbone.Identity.Pool.Creator.Application.Printer;
 using Backbone.Identity.Pool.Creator.Application.RelationshipDistributor;
@@ -19,9 +17,10 @@ public class PoolsGenerator
     private readonly IRelationshipDistributor _relationshipDistributor;
     private readonly IMessageDistributor _messageDistributor;
     private readonly IPrinter _printer;
-    private readonly IList<PoolEntry> _pools;
+    private List<PoolEntry> _pools;
     private readonly PoolsOffset _poolsOffset;
     private readonly ClientCredentials _clientCredentials;
+    private readonly PoolFileRoot _config;
 
     public PoolsGenerator(
         string baseAddress,
@@ -38,14 +37,15 @@ public PoolsGenerator(
         _messageDistributor = messageDistributor;
         _printer = printer;
         _pools = configuration.Pools.ToList();
+        _config = configuration;
 
         MessageDistributorTools.CalculateSentAndReceivedMessages(_pools, configuration.Configuration);
         _poolsOffset = PoolsOffset.CalculatePoolOffsets(_pools.ToArray());
     }
 
     public async Task CreatePools()
     {
-        PoolsOffset.CreateOffsetPools(_pools, _poolsOffset);
+        _pools = PoolsOffset.CreateOffsetPools(_pools, _poolsOffset);
 
         CheckPoolsConfiguration();
 
@@ -70,15 +70,16 @@ public async Task CreatePools()
         //await CreateDatawalletModifications();
 
         OutputAll();
+        _config.Pools = _pools.ToArray();
     }
 
     private uint TrimUnusedRelationships()
     {
-        var allIdentites = _pools.SelectMany(p => p.Identities);
+        var allIdentities = _pools.SelectMany(p => p.Identities);
 
         uint removedCount = 0;
 
-        foreach (var identity in allIdentites)
+        foreach (var identity in allIdentities)
         {
             for (var i = 0; i < identity.IdentitiesToEstablishRelationshipsWith.Count; i++)
             {
@@ -122,7 +123,7 @@ private void CreateFakeIdentities()
         {
             for (uint i = 0; i < pool.Amount; i++)
             {
-                var createdIdentity = new Identity(new("USR" + PasswordHelper.GeneratePassword(8, 8), PasswordHelper.GeneratePassword(18, 24)), "ID1" + PasswordHelper.GeneratePassword(16, 16), "DVC" + PasswordHelper.GeneratePassword(8, 8), pool, uon++);
+                var createdIdentity = new Identity(new("USR" + PasswordHelper.GeneratePassword(8, 8), PasswordHelper.GeneratePassword(18, 24)), "ID1" + PasswordHelper.GeneratePassword(16, 16), "DVC" + PasswordHelper.GeneratePassword(8, 8), pool, i+1, uon++);
 
                 if (pool.NumberOfDevices > 1)
                 {

performance-tests/Identity.Pool.Creator/PoolsGenerator/SimulatedAnnealingGenerator.cs

@@ -1,7 +1,10 @@
-using System.Diagnostics;
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime;
 using Backbone.ConsumerApi.Sdk.Authentication;
 using Backbone.Identity.Pool.Creator.Application.Printer;
 using Backbone.Identity.Pool.Creator.PoolsFile;
+using Backbone.Identity.Pool.Creator.Tools;
 using Backbone.Tooling;
 using Math = System.Math;
 
@@ -26,7 +29,6 @@ public class SimulatedAnnealingPoolsGenerator
     private readonly Dictionary<uint, Identity> _appIdentitiesDictionary;
     private readonly Dictionary<uint, Identity> _connectorIdentitiesDictionary;
 
-
     private const bool P = false;
 
     public SimulatedAnnealingPoolsGenerator(
@@ -41,51 +43,51 @@ public SimulatedAnnealingPoolsGenerator(
         _printer = printer;
         Pools = configuration.Pools.ToList();
 
-        if(!configuration.Pools.SelectMany(p => p.Identities).Any()) CreateIdentities();
+        if (!configuration.Pools.SelectMany(p => p.Identities).Any()) CreateIdentities();
         Pools.AddUonsIfMissing();
 
         _localRandom = new Random();
         Identities = Pools.SelectMany(p => p.Identities).ToList();
         _identitiesDictionary = Identities.ToDictionary(i => i.Uon);
         _appIdentitiesDictionary = Identities.Where(i => i.Pool.IsApp()).ToDictionary(i => i.Uon);
         _connectorIdentitiesDictionary = Identities.Where(i => i.Pool.IsConnector()).ToDictionary(i => i.Uon);
+        _connectorMessageRatio = configuration.Configuration.MessagesSentByConnectorRatio;
     }
 
     private readonly Random _localRandom;
+    private readonly decimal _connectorMessageRatio;
 
     public async Task CreatePools()
     {
         Generate();
-
-        //var (success, messages) = Pools.CheckSolution();
-        //if (!success)
-        //{
-        //    Console.WriteLine("Solution validation failed.");
-
-        //    foreach (var message in messages)
-        //    {
-        //        Console.WriteLine(message);
-        //    }
-        //}
-
-        //_printer.PrintRelationships(Pools, summaryOnly: false);
-        //_printer.PrintMessages(Pools, summaryOnly: false);
     }
 
-    public void Generate(double initialTemperature = 20d, ulong maxIterations = 5000)
+    public void Generate(double initialTemperature = 20d, ulong maxIterations = 20000)
     {
+        var progress = new ProgressBar(Convert.ToInt64(maxIterations));
         var currentSolution = GenerateSolutionFromPools(Pools);
-        
-        var currentScore = CalculateCore(currentSolution, _identitiesDictionary);
+
+        var currentScore = CalculateScore(currentSolution, _identitiesDictionary);
         var temperature = initialTemperature;
 
         for (ulong i = 0; i < maxIterations; i++)
         {
             if (P) Console.Write($"Temp: {temperature:F3} Score:{currentScore}, solution m: {currentSolution.GetSentMessagesCount():D4}, r:{currentSolution.GetRelationshipCount():D4}. Next action: ");
 
-            var nextSolution = GetNextState(currentSolution);
-            if (nextSolution is null) continue;
-            var nextScore = CalculateCore(nextSolution, _identitiesDictionary);
+            var solutions = new ConcurrentBag<SolutionRepresentation>();
+            Parallel.For(0, Environment.ProcessorCount, count => {
+                var nextSolution = currentSolution.Clone() as SolutionRepresentation;
+                for (var si = 0; si < 50; si++)
+                {
+                    nextSolution = GetNextState(nextSolution, _connectorMessageRatio);
+                    if (P && si != 9) Console.Write(", ");
+                }
+
+                if (nextSolution != null) solutions.Add(nextSolution);
+            });
+            var nextSolution = solutions.OrderByDescending(s => CalculateScore(s, _identitiesDictionary)).First();
+
+            var nextScore = CalculateScore(nextSolution, _identitiesDictionary);
 
             if (P) Console.Write($" Next score is {nextScore}");
 
@@ -115,7 +117,9 @@ public void Generate(double initialTemperature = 20d, ulong maxIterations = 5000
                 }
             }
 
+            currentSolution.IterationCount = i;
             temperature = initialTemperature * Math.Pow(1 / initialTemperature, i / (double)maxIterations);
+            progress.Increment();
         }
 
         currentSolution.Print(_identitiesDictionary, Pools);
@@ -144,15 +148,15 @@ private SolutionRepresentation GenerateSolutionFromPools(List<PoolEntry> pools)
         return res;
     }
 
-    private SolutionRepresentation? GetNextState(SolutionRepresentation currentSolution)
+    private SolutionRepresentation? GetNextState(SolutionRepresentation currentSolution, decimal connectorMessageRatio = 0.75m)
     {
         if (currentSolution.Clone() is not SolutionRepresentation solution || solution.GetType() != typeof(SolutionRepresentation))
             return null;
 
         if (_localRandom.NextBoolean())
         {
             // Will mess with messages
-            if (_localRandom.NextDouble() > 0.80)
+            if (_localRandom.NextDouble() > 0.70)
             {
                 // will remove a message
                 solution.RemoveRandomMessage();
@@ -161,18 +165,36 @@ private SolutionRepresentation GenerateSolutionFromPools(List<PoolEntry> pools)
             else
             {
                 // will add a message
-                if (_localRandom.NextBoolean())
-                    solution.SendMessage(_localRandom.GetRandomElement(_appIdentitiesDictionary).Uon, _localRandom.GetRandomElement(_connectorIdentitiesDictionary).Uon);
+                if (_localRandom.NextDouble() > Convert.ToDouble(connectorMessageRatio))
+                {
+                    // app to connector
+                    var appIdentity = _localRandom.GetRandomElement(_appIdentitiesDictionary);
+                    var candidates = solution.GetRelationshipsAndMessageSentCountByIdentity(appIdentity.Uon).ToList();
+                    if (!candidates.Any())
+                        return GetNextState(currentSolution, connectorMessageRatio);
+                    var orderedCandidates = candidates.OrderByDescending(it => appIdentity.Pool.NumberOfSentMessages - it.messageCount);
+                    var selectedCandidate = orderedCandidates.FirstOrDefault().relatedIdentity;
+                    solution.SendMessage(appIdentity.Uon, selectedCandidate);
+                }
                 else
-                    solution.SendMessage(_localRandom.GetRandomElement(_connectorIdentitiesDictionary).Uon, _localRandom.GetRandomElement(_appIdentitiesDictionary).Uon);
+                {
+                    // connector to app
+                    var connectorIdentity = _localRandom.GetRandomElement(_connectorIdentitiesDictionary);
+                    var candidates = solution.GetRelationshipsAndMessageSentCountByIdentity(connectorIdentity.Uon).ToList();
+                    if (!candidates.Any())
+                        return GetNextState(currentSolution, connectorMessageRatio);
+                    var orderedCandidates = candidates.OrderByDescending(it => connectorIdentity.Pool.NumberOfSentMessages - it.messageCount);
+                    var selectedCandidate = orderedCandidates.FirstOrDefault().relatedIdentity;
+                    solution.SendMessage(connectorIdentity.Uon, selectedCandidate);
+                }
 
                 if (P) Console.Write("add msg");
             }
         }
         else
         {
             // Will mess with relationships
-            if (_localRandom.NextDouble() > 0.80)
+            if (_localRandom.NextDouble() > 0.70)
             {
                 // will remove a relationship
                 solution.RemoveRandomRelationship();
@@ -205,28 +227,32 @@ private SolutionRepresentation GenerateSolutionFromPools(List<PoolEntry> pools)
 
     private const double TOLERANCE = 1.05;
 
-    private long CalculateCore(SolutionRepresentation solution, IDictionary<uint, Identity> identities)
+    private long CalculateScore(SolutionRepresentation solution, IDictionary<uint, Identity> identities)
     {
-        var relationshipsTarget = Pools.ExpectedNumberOfRelationships();
+        var relationshipsTarget = Pools.ExpectedNumberOfRelationships(noError: false);
         var sentMessagesTarget = Pools.ExpectedNumberOfSentMessages();
 
         var messagesScore = 0;
-        var messageSentCountByIdentity = solution.GetMessageSendCountByIdentity();
+        var sentAndReceivedMessageCountByIdentityDictionary = solution.GetSentAndReceivedMessageCountByIdentityDictionary();
 
-        foreach (var (identity, messageSentCount) in messageSentCountByIdentity)
+        foreach (var (identity, (messageSentCount, messageReceivedCount)) in sentAndReceivedMessageCountByIdentityDictionary)
         {
 
-            var diff = Convert.ToInt32(messageSentCount) - Convert.ToInt32(identities[identity].Pool.NumberOfSentMessages);
-            if (diff > 0)
-            {
-                messagesScore -= 3 * diff;
-            }
+            var sentDiff = Convert.ToInt32(messageSentCount) - Convert.ToInt32(identities[identity].Pool.NumberOfSentMessages);
+            var receivedDiff = Convert.ToInt32(messageReceivedCount) - Convert.ToInt32(identities[identity].Pool.NumberOfReceivedMessages);
+
+            if (identities[identity].Pool.NumberOfSentMessages == 0 || sentDiff > 4)
+                messagesScore -= 10 * sentDiff;
+            if (identities[identity].Pool.NumberOfReceivedMessages == 0 || receivedDiff > 4)
+                messagesScore -= 10 * receivedDiff;
+
         }
 
         var invalidRelationshipCount = solution.GetInvalidRelationshipCount(_identitiesDictionary);
-        var validRelationshipCount = solution.GetRelationshipCount() - invalidRelationshipCount;
 
-        return -8 * invalidRelationshipCount + 3 * validRelationshipCount - 4 * Math.Abs(relationshipsTarget - solution.GetRelationshipCount()) - 8 * Math.Abs(sentMessagesTarget - solution.GetSentMessagesCount()) + messagesScore;
+        return
+        -8 * invalidRelationshipCount - 4 * Math.Abs(relationshipsTarget - solution.GetRelationshipCount())
+        - 4 * Math.Abs(sentMessagesTarget - solution.GetSentMessagesCount()) + messagesScore;
     }
 
     private SolutionRepresentation GenerateInitialSolution()
@@ -276,6 +302,8 @@ public class SolutionRepresentation : ICloneable
     /// <see cref="Identity.Uon"/>
     private Dictionary<(uint a, uint b), uint> RaM { get; } = new();
 
+    public ulong IterationCount { get; set; }
+
     private readonly Random _localRandom = new();
 
     public SolutionRepresentation(Dictionary<(uint a, uint b), uint>? ram = null, Stopwatch? createdAt = null, (ulong relationshipCount, ulong messagesCount)? counters = null)
@@ -410,6 +438,7 @@ public void Print(Dictionary<uint, Identity> identities, List<PoolEntry> pools)
     {
         Console.WriteLine(" ========= SOLUTION OUTPUT =========");
         Console.WriteLine($" =====> EXECUTION TIME: {GetTimeSinceStart()}");
+        Console.WriteLine($" ====> ITERATION COUNT: {IterationCount + 1}");
         Console.WriteLine($" ==> Expected relationships: {pools.ExpectedNumberOfRelationships()}");
         Console.WriteLine($" ==> Expected sent messages: {pools.ExpectedNumberOfSentMessages()}");
         Console.WriteLine(" =====> ESTABLISHED RELATIONSHIPS");
@@ -452,19 +481,27 @@ public uint GetNumberOfRelatedIdentities(uint identity)
         });
     }
 
-    public Dictionary<uint, uint> GetMessageSendCountByIdentity()
+    public Dictionary<uint, (uint sent, uint received)> GetSentAndReceivedMessageCountByIdentityDictionary()
     {
-        var res = new Dictionary<uint, uint>();
+        var res = new Dictionary<uint, (uint sent, uint received)>();
+
         foreach (var ((a, b), c) in RaM)
         {
-            if (!res.TryAdd(a, c))
-            {
-                res[a] += c;
-            }
+            res.TryAdd(a, (0, 0));
+            res.TryAdd(b, (0, 0));
+
+            res[a] = (res[a].sent + c, res[a].received);
+            res[b] = (res[b].sent, res[b].received + c);
+
         }
 
         return res;
     }
+
+    public IEnumerable<(uint relatedIdentity, uint messageCount)> GetRelationshipsAndMessageSentCountByIdentity(uint uon)
+    {
+        return RaM.Where(it => it.Key.a == uon).Select(it => (it.Key.b, it.Value)).ToList();
+    }
 }
 
 public static class RandomMethodExtensions