Use Pipes for host and benchmark process communication

src/BenchmarkDotNet.Diagnostics.Windows/ConcurrencyVisualizerProfiler.cs

@@ -60,8 +60,6 @@ public void DisplayResults(ILogger logger)
             logger.WriteLineInfo("DO remember that this Diagnoser just tries to mimic the CVCollectionCmd.exe and you need to have Visual Studio with Concurrency Visualizer plugin installed to visualize the data.");
         }
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true;
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
             etwProfiler.Handle(signal, parameters);

src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs

@@ -21,7 +21,6 @@ namespace BenchmarkDotNet.Diagnostics.Windows
         protected readonly Dictionary<BenchmarkCase, int> BenchmarkToProcess = new Dictionary<BenchmarkCase, int>();
         protected readonly ConcurrentDictionary<int, TStats> StatsPerProcess = new ConcurrentDictionary<int, TStats>();
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true;
         public virtual RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.ExtraRun;
 
         public virtual IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();

src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs

@@ -60,8 +60,6 @@ public EtwProfiler(EtwProfilerConfig config)
         public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
             => HardwareCounters.Validate(validationParameters, mandatory: false);
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false;
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
             // it's crucial to start the trace before the process starts and stop it after the benchmarked process stops to have all of the necessary events in the trace file!

src/BenchmarkDotNet.Diagnostics.Windows/NativeMemoryProfiler.cs

@@ -46,8 +46,6 @@ public void DisplayResults(ILogger resultLogger)
                 resultLogger.Write(line.Kind, line.Text);
         }
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => etwProfiler.RequiresBlockingAcknowledgments(benchmarkCase);
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters) => etwProfiler.Handle(signal, parameters);
 
         public RunMode GetRunMode(BenchmarkCase benchmarkCase) => etwProfiler.GetRunMode(benchmarkCase);

src/BenchmarkDotNet/Code/CodeGenerator.cs

@@ -79,9 +79,6 @@ internal static string Generate(BuildPartition buildPartition)
             else if (buildPartition.IsWasm)
                 extraDefines.Add("#define WASM");
 
-            if (buildPartition.NoAcknowledgments)
-                extraDefines.Add("#define NO_ACK");
-
             string benchmarkProgramContent = new SmartStringBuilder(ResourceHelper.LoadTemplate("BenchmarkProgram.txt"))
                 .Replace("$ShadowCopyDefines$", useShadowCopy ? "#define SHADOWCOPY" : null).Replace("$ShadowCopyFolderPath$", shadowCopyFolderPath)
                 .Replace("$ExtraDefines$", string.Join(Environment.NewLine, extraDefines))

src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs

@@ -31,9 +31,6 @@ public IEnumerable<IExporter> Exporters
         public IEnumerable<IAnalyser> Analysers
             => diagnosers.SelectMany(diagnoser => diagnoser.Analysers);
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase)
-            => diagnosers.Any(diagnoser => diagnoser.RequiresBlockingAcknowledgments(benchmarkCase));
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
             foreach (var diagnoser in diagnosers)

src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs

@@ -69,8 +69,6 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
             }
         }
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true;
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
             if (signal != HostSignal.BeforeAnythingElse)

src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs

@@ -20,8 +20,6 @@ public interface IDiagnoser
 
         RunMode GetRunMode(BenchmarkCase benchmarkCase);
 
-        bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase);
-
         void Handle(HostSignal signal, DiagnoserActionParameters parameters);
 
         IEnumerable<Metric> ProcessResults(DiagnoserResults results);

src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs

@@ -30,7 +30,6 @@ public void DisplayResults(ILogger logger) { }
         public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => Array.Empty<ValidationError>();
 
         // the action takes places in other process, and the values are gathered by Engine
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false;
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
         public IEnumerable<Metric> ProcessResults(DiagnoserResults diagnoserResults)

src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs

@@ -29,8 +29,6 @@ public void DisplayResults(ILogger logger) { }
 
         public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false;
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
         public IEnumerable<Metric> ProcessResults(DiagnoserResults results)

src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs

@@ -22,7 +22,6 @@ public class UnresolvedDiagnoser : IDiagnoser
         public IEnumerable<string> Ids => Array.Empty<string>();
         public IEnumerable<IExporter> Exporters => Array.Empty<IExporter>();
         public IEnumerable<IAnalyser> Analysers => Array.Empty<IAnalyser>();
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false;
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
         public IEnumerable<Metric> ProcessResults(DiagnoserResults _) => Array.Empty<Metric>();
 

src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs

@@ -65,8 +65,6 @@ public RunMode GetRunMode(BenchmarkCase benchmarkCase)
             return RunMode.None;
         }
 
-        public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => GetRunMode(benchmarkCase) != RunMode.None;
-
         public void Handle(HostSignal signal, DiagnoserActionParameters parameters)
         {
             var benchmark = parameters.BenchmarkCase;

src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs

@@ -0,0 +1,79 @@
+using BenchmarkDotNet.Validators;
+using System;
+using System.IO;
+using System.Text;
+using Microsoft.Win32.SafeHandles;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Engines
+{
+    public class AnonymousPipesHost : IHost
+    {
+        internal const string AnonymousPipesDescriptors = "--anonymousPipes";
+        internal static readonly Encoding UTF8NoBOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+
+        private readonly StreamWriter outWriter;
+        private readonly StreamReader inReader;
+
+        public AnonymousPipesHost(string writHandle, string readHandle)
+        {
+            outWriter = new StreamWriter(OpenAnonymousPipe(writHandle, FileAccess.Write), UTF8NoBOM);
+            // Flush the data to the Stream after each write, otherwise the host process will wait for input endlessly!
+            outWriter.AutoFlush = true;
+            inReader = new StreamReader(OpenAnonymousPipe(readHandle, FileAccess.Read), UTF8NoBOM, detectEncodingFromByteOrderMarks: false);
+        }
+
+        private Stream OpenAnonymousPipe(string fileHandle, FileAccess access)
+            => new FileStream(new SafeFileHandle(new IntPtr(int.Parse(fileHandle)), ownsHandle: true), access, bufferSize: 1);
+
+        public void Dispose()
+        {
+            outWriter.Dispose();
+            inReader.Dispose();
+        }
+
+        public void Write(string message) => outWriter.Write(message);
+
+        public void WriteLine() => outWriter.WriteLine();
+
+        public void WriteLine(string message) => outWriter.WriteLine(message);
+
+        public void SendSignal(HostSignal hostSignal)
+        {
+            if (hostSignal == HostSignal.AfterAll)
+            {
+                // Before the last signal is reported and the benchmark process exits,
+                // add an artificial sleep to increase the chance of host process reading all std output.
+                System.Threading.Thread.Sleep(1);
+            }
+
+            WriteLine(Engine.Signals.ToMessage(hostSignal));
+
+            // read the response from Parent process, make the communication blocking
+            string acknowledgment = inReader.ReadLine();
+            if (acknowledgment != Engine.Signals.Acknowledgment)
+                throw new NotSupportedException($"Unknown Acknowledgment: {acknowledgment}");
+        }
+
+        public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}");
+
+        public void ReportResults(RunResults runResults) => runResults.Print(outWriter);
+
+        [PublicAPI] // called from generated code
+        public static bool TryGetFileHandles(string[] args, out string writeHandle, out string readHandle)
+        {
+            for (int i = 0; i < args.Length; i++)
+            {
+                if (args[i] == AnonymousPipesDescriptors)
+                {
+                    writeHandle = args[i + 1]; // IndexOutOfRangeException means a bug (incomplete data)
+                    readHandle = args[i + 2];
+                    return true;
+                }
+            }
+
+            writeHandle = readHandle = null;
+            return false;
+        }
+    }
+}

src/BenchmarkDotNet/Engines/GcStats.cs

@@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Engines
 {
     public struct GcStats : IEquatable<GcStats>
     {
-        internal const string ResultsLinePrefix = "GC: ";
+        internal const string ResultsLinePrefix = "// GC: ";
 
         public static readonly long AllocationQuantum = CalculateAllocationQuantumSize();
 

src/BenchmarkDotNet/Engines/IHost.cs

@@ -1,9 +1,10 @@
-using System.Diagnostics.CodeAnalysis;
+using System;
+using System.Diagnostics.CodeAnalysis;
 
 namespace BenchmarkDotNet.Engines
 {
     [SuppressMessage("ReSharper", "UnusedMember.Global")]
-    public interface IHost
+    public interface IHost : IDisposable
     {
         void Write(string message);
         void WriteLine();

src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs

@@ -1,20 +1,15 @@
 using System;
 using System.IO;
 using BenchmarkDotNet.Validators;
-using JetBrains.Annotations;
 
 namespace BenchmarkDotNet.Engines
 {
-    /* This class is used by wasm because wasm cannot read from the console.
-     * This potentially breaks communication with Diagnosers. */
+    // this class is used only when somebody manually launches benchmarking .exe without providing anonymous pipes file descriptors
     public sealed class NoAcknowledgementConsoleHost : IHost
     {
         private readonly TextWriter outWriter;
 
-        public NoAcknowledgementConsoleHost([NotNull]TextWriter outWriter)
-        {
-            this.outWriter = outWriter ?? throw new ArgumentNullException(nameof(outWriter));
-        }
+        public NoAcknowledgementConsoleHost() => outWriter = Console.Out;
 
         public void Write(string message) => outWriter.Write(message);
 
@@ -27,5 +22,10 @@ public NoAcknowledgementConsoleHost([NotNull]TextWriter outWriter)
         public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}");
 
         public void ReportResults(RunResults runResults) => runResults.Print(outWriter);
+
+        public void Dispose()
+        {
+            // do nothing on purpose - there is no point in closing STD OUT
+        }
     }
 }

src/BenchmarkDotNet/Engines/ThreadingStats.cs

@@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Engines
 {
     public struct ThreadingStats : IEquatable<ThreadingStats>
     {
-        internal const string ResultsLinePrefix = "Threading: ";
+        internal const string ResultsLinePrefix = "// Threading: ";
 
         // BDN targets .NET Standard 2.0, these properties are not part of .NET Standard 2.0, were added in .NET Core 3.0
         private static readonly Func<long> GetCompletedWorkItemCountDelegate = CreateGetterDelegate(typeof(ThreadPool), nameof(CompletedWorkItemCount));

src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs

@@ -11,16 +11,16 @@ internal class AsyncProcessOutputReader : IDisposable
     {
         private readonly Process process;
         private readonly ConcurrentQueue<string> output, error;
+        private readonly bool logOutput, readStandardError;
+        private readonly ILogger logger;
 
         private long status;
-        private bool logOutput;
-        private ILogger logger;
 
-        internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger logger = null)
+        internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger logger = null, bool readStandardError = true)
         {
             if (!process.StartInfo.RedirectStandardOutput)
                 throw new NotSupportedException("set RedirectStandardOutput to true first");
-            if (!process.StartInfo.RedirectStandardError)
+            if (readStandardError && !process.StartInfo.RedirectStandardError)
                 throw new NotSupportedException("set RedirectStandardError to true first");
             if (logOutput && logger == null)
                 throw new ArgumentException($"{nameof(logger)} cannot be null when {nameof(logOutput)} is true");
@@ -31,6 +31,7 @@ internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogg
             status = (long)Status.Created;
             this.logOutput = logOutput;
             this.logger = logger;
+            this.readStandardError = readStandardError;
         }
 
         public void Dispose()
@@ -48,7 +49,9 @@ internal void BeginRead()
             Attach();
 
             process.BeginOutputReadLine();
-            process.BeginErrorReadLine();
+
+            if (readStandardError)
+                process.BeginErrorReadLine();
         }
 
         internal void CancelRead()
@@ -57,7 +60,9 @@ internal void CancelRead()
                 throw new InvalidOperationException("Only a started reader can be stopped");
 
             process.CancelOutputRead();
-            process.CancelErrorRead();
+
+            if (readStandardError)
+                process.CancelErrorRead();
 
             Detach();
         }
@@ -83,13 +88,17 @@ internal void StopRead()
         private void Attach()
         {
             process.OutputDataReceived += ProcessOnOutputDataReceived;
-            process.ErrorDataReceived += ProcessOnErrorDataReceived;
+
+            if (readStandardError)
+                process.ErrorDataReceived += ProcessOnErrorDataReceived;
         }
 
         private void Detach()
         {
             process.OutputDataReceived -= ProcessOnOutputDataReceived;
-            process.ErrorDataReceived -= ProcessOnErrorDataReceived;
+
+            if (readStandardError)
+                process.ErrorDataReceived -= ProcessOnErrorDataReceived;
         }
 
         private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs e)