diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/ConcurrencyVisualizerProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/ConcurrencyVisualizerProfiler.cs index 5a5a76112d..acb6280162 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/ConcurrencyVisualizerProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/ConcurrencyVisualizerProfiler.cs @@ -60,8 +60,6 @@ public void DisplayResults(ILogger logger) logger.WriteLineInfo("DO remember that this Diagnoser just tries to mimic the CVCollectionCmd.exe and you need to have Visual Studio with Concurrency Visualizer plugin installed to visualize the data."); } - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true; - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { etwProfiler.Handle(signal, parameters); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs index b3e62f117d..c52006d7c6 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwDiagnoser.cs @@ -21,7 +21,6 @@ namespace BenchmarkDotNet.Diagnostics.Windows protected readonly Dictionary BenchmarkToProcess = new Dictionary(); protected readonly ConcurrentDictionary StatsPerProcess = new ConcurrentDictionary(); - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true; public virtual RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.ExtraRun; public virtual IEnumerable Exporters => Array.Empty(); diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs index 6f09ba86ff..6641dd5a45 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/EtwProfiler.cs @@ -60,8 +60,6 @@ public EtwProfiler(EtwProfilerConfig config) public IEnumerable Validate(ValidationParameters validationParameters) => HardwareCounters.Validate(validationParameters, mandatory: false); - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false; - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { // it's crucial to start the trace before the process starts and stop it after the benchmarked process stops to have all of the necessary events in the trace file! diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/NativeMemoryProfiler.cs b/src/BenchmarkDotNet.Diagnostics.Windows/NativeMemoryProfiler.cs index b05aa2f772..92bc84417c 100644 --- a/src/BenchmarkDotNet.Diagnostics.Windows/NativeMemoryProfiler.cs +++ b/src/BenchmarkDotNet.Diagnostics.Windows/NativeMemoryProfiler.cs @@ -46,8 +46,6 @@ public void DisplayResults(ILogger resultLogger) resultLogger.Write(line.Kind, line.Text); } - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => etwProfiler.RequiresBlockingAcknowledgments(benchmarkCase); - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) => etwProfiler.Handle(signal, parameters); public RunMode GetRunMode(BenchmarkCase benchmarkCase) => etwProfiler.GetRunMode(benchmarkCase); diff --git a/src/BenchmarkDotNet/Code/CodeGenerator.cs b/src/BenchmarkDotNet/Code/CodeGenerator.cs index e60dfe2699..52ae0e5478 100644 --- a/src/BenchmarkDotNet/Code/CodeGenerator.cs +++ b/src/BenchmarkDotNet/Code/CodeGenerator.cs @@ -79,9 +79,6 @@ internal static string Generate(BuildPartition buildPartition) else if (buildPartition.IsWasm) extraDefines.Add("#define WASM"); - if (buildPartition.NoAcknowledgments) - extraDefines.Add("#define NO_ACK"); - string benchmarkProgramContent = new SmartStringBuilder(ResourceHelper.LoadTemplate("BenchmarkProgram.txt")) .Replace("$ShadowCopyDefines$", useShadowCopy ? "#define SHADOWCOPY" : null).Replace("$ShadowCopyFolderPath$", shadowCopyFolderPath) .Replace("$ExtraDefines$", string.Join(Environment.NewLine, extraDefines)) diff --git a/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs index 202822af77..66675f1cb9 100644 --- a/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs @@ -31,9 +31,6 @@ public IEnumerable Exporters public IEnumerable Analysers => diagnosers.SelectMany(diagnoser => diagnoser.Analysers); - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) - => diagnosers.Any(diagnoser => diagnoser.RequiresBlockingAcknowledgments(benchmarkCase)); - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { foreach (var diagnoser in diagnosers) diff --git a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs index e1e655cd86..211d4a06e2 100644 --- a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs @@ -69,8 +69,6 @@ public IEnumerable Validate(ValidationParameters validationPara } } - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => true; - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { if (signal != HostSignal.BeforeAnythingElse) diff --git a/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs index 8045b8e7fa..355dd852e0 100644 --- a/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/IDiagnoser.cs @@ -20,8 +20,6 @@ public interface IDiagnoser RunMode GetRunMode(BenchmarkCase benchmarkCase); - bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase); - void Handle(HostSignal signal, DiagnoserActionParameters parameters); IEnumerable ProcessResults(DiagnoserResults results); diff --git a/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs index 862e6cc041..b6c2eb12bb 100644 --- a/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs @@ -30,7 +30,6 @@ public void DisplayResults(ILogger logger) { } public IEnumerable Validate(ValidationParameters validationParameters) => Array.Empty(); // the action takes places in other process, and the values are gathered by Engine - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false; public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults diagnoserResults) diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index a1997c23ff..cf19cba1e1 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -29,8 +29,6 @@ public void DisplayResults(ILogger logger) { } public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead; - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false; - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults results) diff --git a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs index 6db34ef072..7ec0498c34 100644 --- a/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/UnresolvedDiagnoser.cs @@ -22,7 +22,6 @@ public class UnresolvedDiagnoser : IDiagnoser public IEnumerable Ids => Array.Empty(); public IEnumerable Exporters => Array.Empty(); public IEnumerable Analysers => Array.Empty(); - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => false; public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { } public IEnumerable ProcessResults(DiagnoserResults _) => Array.Empty(); diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 6b094ae4cd..376788ffae 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -65,8 +65,6 @@ public RunMode GetRunMode(BenchmarkCase benchmarkCase) return RunMode.None; } - public bool RequiresBlockingAcknowledgments(BenchmarkCase benchmarkCase) => GetRunMode(benchmarkCase) != RunMode.None; - public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { var benchmark = parameters.BenchmarkCase; diff --git a/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs b/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs new file mode 100644 index 0000000000..f1d5390a9f --- /dev/null +++ b/src/BenchmarkDotNet/Engines/AnonymousPipesHost.cs @@ -0,0 +1,79 @@ +using BenchmarkDotNet.Validators; +using System; +using System.IO; +using System.Text; +using Microsoft.Win32.SafeHandles; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Engines +{ + public class AnonymousPipesHost : IHost + { + internal const string AnonymousPipesDescriptors = "--anonymousPipes"; + internal static readonly Encoding UTF8NoBOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + private readonly StreamWriter outWriter; + private readonly StreamReader inReader; + + public AnonymousPipesHost(string writHandle, string readHandle) + { + outWriter = new StreamWriter(OpenAnonymousPipe(writHandle, FileAccess.Write), UTF8NoBOM); + // Flush the data to the Stream after each write, otherwise the host process will wait for input endlessly! + outWriter.AutoFlush = true; + inReader = new StreamReader(OpenAnonymousPipe(readHandle, FileAccess.Read), UTF8NoBOM, detectEncodingFromByteOrderMarks: false); + } + + private Stream OpenAnonymousPipe(string fileHandle, FileAccess access) + => new FileStream(new SafeFileHandle(new IntPtr(int.Parse(fileHandle)), ownsHandle: true), access, bufferSize: 1); + + public void Dispose() + { + outWriter.Dispose(); + inReader.Dispose(); + } + + public void Write(string message) => outWriter.Write(message); + + public void WriteLine() => outWriter.WriteLine(); + + public void WriteLine(string message) => outWriter.WriteLine(message); + + public void SendSignal(HostSignal hostSignal) + { + if (hostSignal == HostSignal.AfterAll) + { + // Before the last signal is reported and the benchmark process exits, + // add an artificial sleep to increase the chance of host process reading all std output. + System.Threading.Thread.Sleep(1); + } + + WriteLine(Engine.Signals.ToMessage(hostSignal)); + + // read the response from Parent process, make the communication blocking + string acknowledgment = inReader.ReadLine(); + if (acknowledgment != Engine.Signals.Acknowledgment) + throw new NotSupportedException($"Unknown Acknowledgment: {acknowledgment}"); + } + + public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}"); + + public void ReportResults(RunResults runResults) => runResults.Print(outWriter); + + [PublicAPI] // called from generated code + public static bool TryGetFileHandles(string[] args, out string writeHandle, out string readHandle) + { + for (int i = 0; i < args.Length; i++) + { + if (args[i] == AnonymousPipesDescriptors) + { + writeHandle = args[i + 1]; // IndexOutOfRangeException means a bug (incomplete data) + readHandle = args[i + 2]; + return true; + } + } + + writeHandle = readHandle = null; + return false; + } + } +} diff --git a/src/BenchmarkDotNet/Engines/ConsoleHost.cs b/src/BenchmarkDotNet/Engines/ConsoleHost.cs deleted file mode 100644 index 738555807f..0000000000 --- a/src/BenchmarkDotNet/Engines/ConsoleHost.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.IO; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Engines -{ - public sealed class ConsoleHost : IHost - { - private readonly TextWriter outWriter; - private readonly StreamReader inReader; - - public ConsoleHost([NotNull]TextWriter outWriter, [NotNull]StreamReader inReader) - { - this.outWriter = outWriter ?? throw new ArgumentNullException(nameof(outWriter)); - this.inReader = inReader ?? throw new ArgumentNullException(nameof(inReader)); - } - - public void Write(string message) => outWriter.Write(message); - - public void WriteLine() => outWriter.WriteLine(); - - public void WriteLine(string message) => outWriter.WriteLine(message); - - public void SendSignal(HostSignal hostSignal) - { - WriteLine(Engine.Signals.ToMessage(hostSignal)); - - // read the response from Parent process, make the communication blocking - // I did not use Mutexes because they are not supported for Linux/MacOs for .NET Core - // this solution is stupid simple and it works - string acknowledgment = inReader.ReadLine(); - if (acknowledgment != Engine.Signals.Acknowledgment) - throw new NotSupportedException($"Unknown Acknowledgment: {acknowledgment}"); - } - - public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}"); - - public void ReportResults(RunResults runResults) => runResults.Print(outWriter); - } -} diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index 2f2552b4d0..1d3812173e 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -9,7 +9,7 @@ namespace BenchmarkDotNet.Engines { public struct GcStats : IEquatable { - internal const string ResultsLinePrefix = "GC: "; + internal const string ResultsLinePrefix = "// GC: "; public static readonly long AllocationQuantum = CalculateAllocationQuantumSize(); diff --git a/src/BenchmarkDotNet/Engines/IHost.cs b/src/BenchmarkDotNet/Engines/IHost.cs index 2e6895ebc3..e57ee2172b 100644 --- a/src/BenchmarkDotNet/Engines/IHost.cs +++ b/src/BenchmarkDotNet/Engines/IHost.cs @@ -1,9 +1,10 @@ -using System.Diagnostics.CodeAnalysis; +using System; +using System.Diagnostics.CodeAnalysis; namespace BenchmarkDotNet.Engines { [SuppressMessage("ReSharper", "UnusedMember.Global")] - public interface IHost + public interface IHost : IDisposable { void Write(string message); void WriteLine(); diff --git a/src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs b/src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs index 4be570e933..fd97997aa0 100644 --- a/src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs +++ b/src/BenchmarkDotNet/Engines/NoAcknowledgementConsoleHost.cs @@ -1,20 +1,15 @@ using System; using System.IO; using BenchmarkDotNet.Validators; -using JetBrains.Annotations; namespace BenchmarkDotNet.Engines { - /* This class is used by wasm because wasm cannot read from the console. - * This potentially breaks communication with Diagnosers. */ + // this class is used only when somebody manually launches benchmarking .exe without providing anonymous pipes file descriptors public sealed class NoAcknowledgementConsoleHost : IHost { private readonly TextWriter outWriter; - public NoAcknowledgementConsoleHost([NotNull]TextWriter outWriter) - { - this.outWriter = outWriter ?? throw new ArgumentNullException(nameof(outWriter)); - } + public NoAcknowledgementConsoleHost() => outWriter = Console.Out; public void Write(string message) => outWriter.Write(message); @@ -27,5 +22,10 @@ public NoAcknowledgementConsoleHost([NotNull]TextWriter outWriter) public void SendError(string message) => outWriter.WriteLine($"{ValidationErrorReporter.ConsoleErrorPrefix} {message}"); public void ReportResults(RunResults runResults) => runResults.Print(outWriter); + + public void Dispose() + { + // do nothing on purpose - there is no point in closing STD OUT + } } } diff --git a/src/BenchmarkDotNet/Engines/ThreadingStats.cs b/src/BenchmarkDotNet/Engines/ThreadingStats.cs index d196577b75..6afa5fec11 100644 --- a/src/BenchmarkDotNet/Engines/ThreadingStats.cs +++ b/src/BenchmarkDotNet/Engines/ThreadingStats.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Engines { public struct ThreadingStats : IEquatable { - internal const string ResultsLinePrefix = "Threading: "; + internal const string ResultsLinePrefix = "// Threading: "; // BDN targets .NET Standard 2.0, these properties are not part of .NET Standard 2.0, were added in .NET Core 3.0 private static readonly Func GetCompletedWorkItemCountDelegate = CreateGetterDelegate(typeof(ThreadPool), nameof(CompletedWorkItemCount)); diff --git a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs index 694bcf6352..480b7513cc 100644 --- a/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs +++ b/src/BenchmarkDotNet/Loggers/AsyncProcessOutputReader.cs @@ -11,16 +11,16 @@ internal class AsyncProcessOutputReader : IDisposable { private readonly Process process; private readonly ConcurrentQueue output, error; + private readonly bool logOutput, readStandardError; + private readonly ILogger logger; private long status; - private bool logOutput; - private ILogger logger; - internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger logger = null) + internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogger logger = null, bool readStandardError = true) { if (!process.StartInfo.RedirectStandardOutput) throw new NotSupportedException("set RedirectStandardOutput to true first"); - if (!process.StartInfo.RedirectStandardError) + if (readStandardError && !process.StartInfo.RedirectStandardError) throw new NotSupportedException("set RedirectStandardError to true first"); if (logOutput && logger == null) throw new ArgumentException($"{nameof(logger)} cannot be null when {nameof(logOutput)} is true"); @@ -31,6 +31,7 @@ internal AsyncProcessOutputReader(Process process, bool logOutput = false, ILogg status = (long)Status.Created; this.logOutput = logOutput; this.logger = logger; + this.readStandardError = readStandardError; } public void Dispose() @@ -48,7 +49,9 @@ internal void BeginRead() Attach(); process.BeginOutputReadLine(); - process.BeginErrorReadLine(); + + if (readStandardError) + process.BeginErrorReadLine(); } internal void CancelRead() @@ -57,7 +60,9 @@ internal void CancelRead() throw new InvalidOperationException("Only a started reader can be stopped"); process.CancelOutputRead(); - process.CancelErrorRead(); + + if (readStandardError) + process.CancelErrorRead(); Detach(); } @@ -83,13 +88,17 @@ internal void StopRead() private void Attach() { process.OutputDataReceived += ProcessOnOutputDataReceived; - process.ErrorDataReceived += ProcessOnErrorDataReceived; + + if (readStandardError) + process.ErrorDataReceived += ProcessOnErrorDataReceived; } private void Detach() { process.OutputDataReceived -= ProcessOnOutputDataReceived; - process.ErrorDataReceived -= ProcessOnErrorDataReceived; + + if (readStandardError) + process.ErrorDataReceived -= ProcessOnErrorDataReceived; } private void ProcessOnOutputDataReceived(object sender, DataReceivedEventArgs e) diff --git a/src/BenchmarkDotNet/Loggers/Broker.cs b/src/BenchmarkDotNet/Loggers/Broker.cs new file mode 100644 index 0000000000..e915d61eca --- /dev/null +++ b/src/BenchmarkDotNet/Loggers/Broker.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Pipes; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Loggers +{ + internal class Broker + { + private readonly ILogger logger; + private readonly IDiagnoser diagnoser; + private readonly AnonymousPipeServerStream inputFromBenchmark, acknowledgments; + private readonly DiagnoserActionParameters diagnoserActionParameters; + + public Broker(ILogger logger, Process process, IDiagnoser diagnoser, + BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, AnonymousPipeServerStream inputFromBenchmark, AnonymousPipeServerStream acknowledgments) + { + this.logger = logger; + this.diagnoser = diagnoser; + this.inputFromBenchmark = inputFromBenchmark; + this.acknowledgments = acknowledgments; + diagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); + + Results = new List(); + PrefixedOutput = new List(); + } + + internal List Results { get; } + + internal List PrefixedOutput { get; } + + internal void ProcessData() + { + using StreamReader reader = new (inputFromBenchmark, AnonymousPipesHost.UTF8NoBOM, detectEncodingFromByteOrderMarks: false); + using StreamWriter writer = new (acknowledgments, AnonymousPipesHost.UTF8NoBOM, bufferSize: 1); + // Flush the data to the Stream after each write, otherwise the client will wait for input endlessly! + writer.AutoFlush = true; + string line = null; + + while ((line = reader.ReadLine()) is not null) + { + // TODO: implement Silent mode here + logger.WriteLine(LogKind.Default, line); + + if (!line.StartsWith("//")) + { + Results.Add(line); + } + else if (Engine.Signals.TryGetSignal(line, out var signal)) + { + diagnoser?.Handle(signal, diagnoserActionParameters); + + writer.WriteLine(Engine.Signals.Acknowledgment); + + if (signal == HostSignal.AfterAll) + { + // we have received the last signal so we can stop reading from the pipe + // if the process won't exit after this, its hung and needs to be killed + return; + } + } + else if (!string.IsNullOrEmpty(line)) + { + PrefixedOutput.Add(line); + } + } + } + } +} diff --git a/src/BenchmarkDotNet/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs b/src/BenchmarkDotNet/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs deleted file mode 100644 index 6ce058d9d6..0000000000 --- a/src/BenchmarkDotNet/Loggers/SynchronousProcessOutputLoggerWithDiagnoser.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Running; - -namespace BenchmarkDotNet.Loggers -{ - internal class SynchronousProcessOutputLoggerWithDiagnoser - { - private readonly ILogger logger; - private readonly Process process; - private readonly IDiagnoser diagnoser; - private readonly DiagnoserActionParameters diagnoserActionParameters; - - public SynchronousProcessOutputLoggerWithDiagnoser(ILogger logger, Process process, IDiagnoser diagnoser, - BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, bool noAcknowledgments) - { - if (!process.StartInfo.RedirectStandardOutput) - throw new NotSupportedException("set RedirectStandardOutput to true first"); - if (!(process.StartInfo.RedirectStandardInput || benchmarkCase.GetRuntime() is WasmRuntime || noAcknowledgments)) - throw new NotSupportedException("set RedirectStandardInput to true first"); - - this.logger = logger; - this.process = process; - this.diagnoser = diagnoser; - diagnoserActionParameters = new DiagnoserActionParameters(process, benchmarkCase, benchmarkId); - - LinesWithResults = new List(); - LinesWithExtraOutput = new List(); - } - - internal List LinesWithResults { get; } - - internal List LinesWithExtraOutput { get; } - - internal void ProcessInput() - { - // Peek -1 or 0 can indicate internal error. - while (!process.StandardOutput.EndOfStream && process.StandardOutput.Peek() > 0) - { - // ReadLine() can actually return string.Empty and null as valid values. - string line = process.StandardOutput.ReadLine(); - - if (line == null) - continue; - - logger.WriteLine(LogKind.Default, line); - - if (!line.StartsWith("//")) - { - LinesWithResults.Add(line); - } - else if (Engine.Signals.TryGetSignal(line, out var signal)) - { - diagnoser?.Handle(signal, diagnoserActionParameters); - - if (process.StartInfo.RedirectStandardInput) - { - process.StandardInput.WriteLine(Engine.Signals.Acknowledgment); - } - - if (signal == HostSignal.AfterAll) - { - // we have received the last signal so we can stop reading the output - // if the process won't exit after this, its hung and needs to be killed - return; - } - } - else if (!string.IsNullOrEmpty(line)) - { - LinesWithExtraOutput.Add(line); - } - } - } - } -} diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index 6b049adb33..5b743a3129 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -76,7 +76,12 @@ private static bool IsAotMethod() #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] #endif - internal static bool IsWindows() => IsOSPlatform(OSPlatform.Windows); + internal static bool IsWindows() => +#if NET6_0_OR_GREATER + OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs +#else + IsOSPlatform(OSPlatform.Windows); +#endif #if NET6_0_OR_GREATER [System.Runtime.Versioning.SupportedOSPlatformGuard("linux")] diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReportExtensions.cs b/src/BenchmarkDotNet/Reports/BenchmarkReportExtensions.cs index 6681ccb210..4f163dc66e 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReportExtensions.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReportExtensions.cs @@ -24,7 +24,7 @@ private static string GetInfoFromOutput(this BenchmarkReport report, string pref { return ( from executeResults in report.ExecuteResults - from extraOutputLine in executeResults.ExtraOutput.Where(line => line.StartsWith(prefix)) + from extraOutputLine in executeResults.PrefixedLines.Where(line => line.StartsWith(prefix)) select extraOutputLine.Substring(prefix.Length)).FirstOrDefault(); } } diff --git a/src/BenchmarkDotNet/Running/BenchmarkId.cs b/src/BenchmarkDotNet/Running/BenchmarkId.cs index 1d79657bca..acfc89ce85 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkId.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkId.cs @@ -1,4 +1,5 @@ using System; +using BenchmarkDotNet.Engines; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using JetBrains.Annotations; @@ -31,6 +32,8 @@ public BenchmarkId(int value, BenchmarkCase benchmarkCase) public string ToArguments() => $"--benchmarkName {FullBenchmarkName.Escape()} --job {JobId.Escape()} --benchmarkId {Value}"; + public string ToArguments(string fromBenchmark, string toBenchmark) => $"{AnonymousPipesHost.AnonymousPipesDescriptors} {fromBenchmark} {toBenchmark} {ToArguments()}"; + public override string ToString() => Value.ToString(); private static string GetBenchmarkName(BenchmarkCase benchmark) diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index 3a98ea5a56..5275acd61b 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -68,11 +68,6 @@ public BuildPartition(BenchmarkBuildInfo[] benchmarks, IResolver resolver) public bool GenerateMSBuildBinLog { get; } - public bool NoAcknowledgments - => !Benchmarks - .Any(bennchmark => bennchmark.Config.GetDiagnosers() - .Any(diagnoser => diagnoser.RequiresBlockingAcknowledgments(bennchmark.BenchmarkCase))); - public override string ToString() => RepresentativeBenchmarkCase.Job.DisplayInfo; private static string GetResolvedAssemblyLocation(Assembly assembly) => diff --git a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt index fe53e9e261..65dc0a24e6 100644 --- a/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt +++ b/src/BenchmarkDotNet/Templates/BenchmarkProgram.txt @@ -24,12 +24,11 @@ namespace BenchmarkDotNet.Autogenerated private static System.Int32 AfterAssemblyLoadingAttached(System.String[] args) { - BenchmarkDotNet.Engines.IHost host = // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it -#if WASM || NO_ACK - new BenchmarkDotNet.Engines.NoAcknowledgementConsoleHost(System.Console.Out); -#else - new BenchmarkDotNet.Engines.ConsoleHost(System.Console.Out, new System.IO.StreamReader(System.Console.OpenStandardInput(), System.Text.Encoding.UTF8)); -#endif + BenchmarkDotNet.Engines.IHost host; // this variable name is used by CodeGenerator.GetCoreRtSwitch, do NOT change it + if (BenchmarkDotNet.Engines.AnonymousPipesHost.TryGetFileHandles(args, out System.String writeHandle, out System.String readHandle)) + host = new BenchmarkDotNet.Engines.AnonymousPipesHost(writeHandle, readHandle); + else + host = new BenchmarkDotNet.Engines.NoAcknowledgementConsoleHost(); // the first thing to do is to let diagnosers hook in before anything happens // so all jit-related diagnosers can catch first jit compilation! @@ -81,6 +80,8 @@ namespace BenchmarkDotNet.Autogenerated finally { BenchmarkDotNet.Engines.HostExtensions.AfterAll(host); + + host.Dispose(); } } } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index bc196249cc..9d4cefea0c 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -99,7 +99,7 @@ internal static void LogEnvVars(DotNetCliCommand command) } internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, string workingDirectory, string arguments, - IReadOnlyList environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true) + IReadOnlyList environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true, bool redirectStandardOutput = true) { const string dotnetMultiLevelLookupEnvVarName = "DOTNET_MULTILEVEL_LOOKUP"; @@ -110,12 +110,16 @@ internal static ProcessStartInfo BuildStartInfo(string customDotNetCliPath, stri Arguments = arguments, UseShellExecute = false, CreateNoWindow = true, - RedirectStandardOutput = true, + RedirectStandardOutput = redirectStandardOutput, RedirectStandardError = redirectStandardError, RedirectStandardInput = redirectStandardInput, - StandardOutputEncoding = Encoding.UTF8, }; + if (redirectStandardOutput) + { + startInfo.StandardOutputEncoding = Encoding.UTF8; + } + if (redirectStandardError) // StandardErrorEncoding is only supported when standard error is redirected { startInfo.StandardErrorEncoding = Encoding.UTF8; diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index 282075db8e..403f38bf02 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -1,6 +1,7 @@ using System; using System.Diagnostics; using System.IO; +using System.IO.Pipes; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -43,8 +44,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) executeParameters.Diagnoser, Path.GetFileName(executeParameters.BuildResult.ArtifactsPaths.ExecutablePath), executeParameters.Resolver, - executeParameters.LaunchIndex, - executeParameters.BuildResult.NoAcknowledgments); + executeParameters.LaunchIndex); } finally { @@ -61,28 +61,33 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, IDiagnoser diagnoser, string executableName, IResolver resolver, - int launchIndex, - bool noAcknowledgments) + int launchIndex) { + using AnonymousPipeServerStream inputFromBenchmark = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable); + using AnonymousPipeServerStream acknowledgments = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable); + var startInfo = DotNetCliCommandExecutor.BuildStartInfo( CustomDotNetCliPath, artifactsPaths.BinariesDirectoryPath, - $"{executableName.Escape()} {benchmarkId.ToArguments()}", - redirectStandardInput: !noAcknowledgments, + $"{executableName.Escape()} {benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString())}", + redirectStandardOutput: true, + redirectStandardInput: false, redirectStandardError: false); // #1629 startInfo.SetEnvironmentVariables(benchmarkCase, resolver); - using (var process = new Process { StartInfo = startInfo }) - using (var consoleExitHandler = new ConsoleExitHandler(process, logger)) + using (Process process = new () { StartInfo = startInfo }) + using (ConsoleExitHandler consoleExitHandler = new (process, logger)) + using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) { - var loggerWithDiagnoser = new SynchronousProcessOutputLoggerWithDiagnoser(logger, process, diagnoser, benchmarkCase, benchmarkId, noAcknowledgments); + Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); process.Start(); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); if (benchmarkCase.Job.Environment.HasValue(EnvironmentMode.AffinityCharacteristic)) @@ -90,20 +95,26 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, process.TrySetAffinity(benchmarkCase.Job.Environment.Affinity, logger); } - loggerWithDiagnoser.ProcessInput(); + broker.ProcessData(); if (!process.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) { logger.WriteLineInfo($"// The benchmarking process did not quit within {ExecuteParameters.ProcessExitTimeout.TotalSeconds} seconds, it's going to get force killed now."); + processOutputReader.CancelRead(); consoleExitHandler.KillProcessTree(); } + else + { + processOutputReader.StopRead(); + } return new ExecuteResult(true, process.HasExited ? process.ExitCode : null, process.Id, - loggerWithDiagnoser.LinesWithResults, - loggerWithDiagnoser.LinesWithExtraOutput, + broker.Results, + broker.PrefixedOutput, + processOutputReader.GetOutputLines(), launchIndex); } } diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index 6412fd0d29..8eb0576a76 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -1,6 +1,7 @@ using System; using System.Diagnostics; using System.IO; +using System.IO.Pipes; using System.Linq; using System.Text; using BenchmarkDotNet.Characteristics; @@ -24,7 +25,6 @@ public class Executor : IExecutor public ExecuteResult Execute(ExecuteParameters executeParameters) { string exePath = executeParameters.BuildResult.ArtifactsPaths.ExecutablePath; - string args = executeParameters.BenchmarkId.ToArguments(); if (!File.Exists(exePath)) { @@ -32,22 +32,28 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) } return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, executeParameters.BuildResult.ArtifactsPaths, - args, executeParameters.Diagnoser, executeParameters.Resolver, executeParameters.LaunchIndex, executeParameters.BuildResult.NoAcknowledgments); + executeParameters.Diagnoser, executeParameters.Resolver, executeParameters.LaunchIndex); } - private ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, - string args, IDiagnoser diagnoser, IResolver resolver, int launchIndex, bool noAcknowledgments) + private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, + IDiagnoser diagnoser, IResolver resolver, int launchIndex) { try { - using (var process = new Process { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver, noAcknowledgments) }) - using (var consoleExitHandler = new ConsoleExitHandler(process, logger)) + using AnonymousPipeServerStream inputFromBenchmark = new (PipeDirection.In, HandleInheritability.Inheritable); + using AnonymousPipeServerStream acknowledgments = new (PipeDirection.Out, HandleInheritability.Inheritable); + + string args = benchmarkId.ToArguments(inputFromBenchmark.GetClientHandleAsString(), acknowledgments.GetClientHandleAsString()); + + using (Process process = new () { StartInfo = CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver) }) + using (ConsoleExitHandler consoleExitHandler = new (process, logger)) + using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) { - var loggerWithDiagnoser = new SynchronousProcessOutputLoggerWithDiagnoser(logger, process, diagnoser, benchmarkCase, benchmarkId, noAcknowledgments); + Broker broker = new (logger, process, diagnoser, benchmarkCase, benchmarkId, inputFromBenchmark, acknowledgments); diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); - return Execute(process, benchmarkCase, loggerWithDiagnoser, logger, consoleExitHandler, launchIndex); + return Execute(process, benchmarkCase, broker, logger, consoleExitHandler, launchIndex, processOutputReader); } } finally @@ -56,12 +62,13 @@ private ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmark } } - private ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, SynchronousProcessOutputLoggerWithDiagnoser loggerWithDiagnoser, - ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex) + private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, Broker broker, + ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex, AsyncProcessOutputReader processOutputReader) { logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); process.Start(); + processOutputReader.BeginRead(); process.EnsureHighPriority(logger); if (benchmarkCase.Job.Environment.HasValue(EnvironmentMode.AffinityCharacteristic)) @@ -69,37 +76,41 @@ private ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, Sync process.TrySetAffinity(benchmarkCase.Job.Environment.Affinity, logger); } - loggerWithDiagnoser.ProcessInput(); + broker.ProcessData(); if (!process.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) { logger.WriteLineInfo("// The benchmarking process did not quit on time, it's going to get force killed now."); + processOutputReader.CancelRead(); consoleExitHandler.KillProcessTree(); } + else + { + processOutputReader.StopRead(); + } - if (loggerWithDiagnoser.LinesWithResults.Any(line => line.Contains("BadImageFormatException"))) + if (broker.Results.Any(line => line.Contains("BadImageFormatException"))) logger.WriteLineError("You are probably missing AnyCPU in your .csproj file."); return new ExecuteResult(true, process.HasExited ? process.ExitCode : null, process.Id, - loggerWithDiagnoser.LinesWithResults, - loggerWithDiagnoser.LinesWithExtraOutput, + broker.Results, + broker.PrefixedOutput, + processOutputReader.GetOutputLines(), launchIndex); } - private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, - string args, IResolver resolver, bool noAcknowledgments) + private static ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) { var start = new ProcessStartInfo { UseShellExecute = false, RedirectStandardOutput = true, - RedirectStandardInput = !noAcknowledgments, + RedirectStandardInput = false, RedirectStandardError = false, // #1629 CreateNoWindow = true, - StandardOutputEncoding = Encoding.UTF8, // #1713 WorkingDirectory = null // by default it's null }; @@ -108,7 +119,6 @@ private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsP string exePath = artifactsPaths.ExecutablePath; var runtime = benchmarkCase.GetRuntime(); - // TODO: use resolver switch (runtime) { @@ -122,15 +132,6 @@ private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsP start.FileName = mono.CustomPath ?? "mono"; start.Arguments = GetMonoArguments(benchmarkCase.Job, exePath, args, resolver); break; - case WasmRuntime wasm: - start.FileName = wasm.JavaScriptEngine; - start.RedirectStandardInput = false; - - string main_js = runtime.RuntimeMoniker < RuntimeMoniker.WasmNet70 ? "main.js" : "test-main.js"; - - start.Arguments = $"{wasm.JavaScriptEngineArguments} {main_js} -- --run {artifactsPaths.ProgramName}.dll {args} "; - start.WorkingDirectory = artifactsPaths.BinariesDirectoryPath; - break; case MonoAotLLVMRuntime _: start.FileName = exePath; start.Arguments = args; @@ -142,7 +143,7 @@ private ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsP return start; } - private string GetMonoArguments(Job job, string exePath, string args, IResolver resolver) + private static string GetMonoArguments(Job job, string exePath, string args, IResolver resolver) { var arguments = job.HasValue(InfrastructureMode.ArgumentsCharacteristic) ? job.ResolveValue(InfrastructureMode.ArgumentsCharacteristic, resolver).OfType().ToArray() diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index 81ebf2100a..0d7ab30d7d 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -29,7 +29,7 @@ public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger log GenerateProject(buildPartition, artifactsPaths, logger); GenerateBuildScript(buildPartition, artifactsPaths); - return GenerateResult.Success(artifactsPaths, GetArtifactsToCleanup(artifactsPaths), buildPartition.NoAcknowledgments); + return GenerateResult.Success(artifactsPaths, GetArtifactsToCleanup(artifactsPaths)); } catch (Exception ex) { diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitBuilder.cs b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitBuilder.cs index 6381f709f7..f46f5c99b6 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitBuilder.cs @@ -29,8 +29,7 @@ public BuildResult Build(GenerateResult generateResult, BuildPartition buildPart return BuildResult.Success( GenerateResult.Success( new InProcessEmitArtifactsPath(assembly, generateResult.ArtifactsPaths), - generateResult.ArtifactsToCleanup, - generateResult.NoAcknowledgments)); + generateResult.ArtifactsToCleanup)); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitGenerator.cs index 8035f7854d..c169c91903 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.Emit/InProcessEmitGenerator.cs @@ -20,7 +20,7 @@ public GenerateResult GenerateProject( { artifactsPaths = GetArtifactsPaths(buildPartition, rootArtifactsFolderPath); - return GenerateResult.Success(artifactsPaths, new List(), buildPartition.NoAcknowledgments); + return GenerateResult.Success(artifactsPaths, new List()); } catch (Exception ex) { diff --git a/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitGenerator.cs index 91ec5b89fe..cd1b24dbf8 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess.NoEmit/InProcessNoEmitGenerator.cs @@ -13,6 +13,6 @@ public class InProcessNoEmitGenerator : IGenerator { /// returns a success public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath) - => GenerateResult.Success(null, Array.Empty(), buildPartition.NoAcknowledgments); + => GenerateResult.Success(null, Array.Empty()); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessGenerator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessGenerator.cs index 8a062e5ab7..ca9803df55 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessGenerator.cs @@ -13,6 +13,6 @@ public class InProcessGenerator : IGenerator { /// returns a success public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath) - => GenerateResult.Success(null, Array.Empty(), buildPartition.NoAcknowledgments); + => GenerateResult.Success(null, Array.Empty()); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs index cf73c18794..726d69b2bd 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessHost.cs @@ -69,7 +69,6 @@ public InProcessHost(BenchmarkCase benchmarkCase, ILogger logger, IDiagnoser dia /// Text to write. public void WriteLine(string message) => logger.WriteLine(message); - /// Sends notification signal to the host. /// The signal to send. public void SendSignal(HostSignal hostSignal) @@ -97,5 +96,10 @@ public void ReportResults(RunResults runResults) logger.Write(w.GetStringBuilder().ToString()); } } + + public void Dispose() + { + // do nothing on purpose + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs new file mode 100644 index 0000000000..024c8e4242 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs @@ -0,0 +1,114 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Parameters; +using BenchmarkDotNet.Toolchains.Results; +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace BenchmarkDotNet.Toolchains.MonoWasm +{ + internal class WasmExecutor : IExecutor + { + public ExecuteResult Execute(ExecuteParameters executeParameters) + { + string exePath = executeParameters.BuildResult.ArtifactsPaths.ExecutablePath; + + if (!File.Exists(exePath)) + { + return ExecuteResult.CreateFailed(); + } + + return Execute(executeParameters.BenchmarkCase, executeParameters.BenchmarkId, executeParameters.Logger, executeParameters.BuildResult.ArtifactsPaths, + executeParameters.Diagnoser, executeParameters.Resolver, executeParameters.LaunchIndex); + } + + private static ExecuteResult Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, + IDiagnoser diagnoser, IResolver resolver, int launchIndex) + { + try + { + using (Process process = CreateProcess(benchmarkCase, artifactsPaths, benchmarkId.ToArguments(), resolver)) + using (ConsoleExitHandler consoleExitHandler = new (process, logger)) + using (AsyncProcessOutputReader processOutputReader = new (process, logOutput: true, logger, readStandardError: false)) + { + diagnoser?.Handle(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(process, benchmarkCase, benchmarkId)); + + return Execute(process, benchmarkCase, processOutputReader, logger, consoleExitHandler, launchIndex); + } + } + finally + { + diagnoser?.Handle(HostSignal.AfterProcessExit, new DiagnoserActionParameters(null, benchmarkCase, benchmarkId)); + } + } + + private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) + { + WasmRuntime runtime = (WasmRuntime)benchmarkCase.GetRuntime(); + string mainJs = runtime.RuntimeMoniker < RuntimeMoniker.WasmNet70 ? "main.js" : "test-main.js"; + + var start = new ProcessStartInfo + { + FileName = runtime.JavaScriptEngine, + Arguments = $"{runtime.JavaScriptEngineArguments} {mainJs} -- --run {artifactsPaths.ProgramName}.dll {args} ", + WorkingDirectory = artifactsPaths.BinariesDirectoryPath, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardInput = false, // not supported by WASM! + RedirectStandardError = false, // #1629 + CreateNoWindow = true + }; + + start.SetEnvironmentVariables(benchmarkCase, resolver); + + return new Process() { StartInfo = start }; + } + + private static ExecuteResult Execute(Process process, BenchmarkCase benchmarkCase, AsyncProcessOutputReader processOutputReader, + ILogger logger, ConsoleExitHandler consoleExitHandler, int launchIndex) + { + logger.WriteLineInfo($"// Execute: {process.StartInfo.FileName} {process.StartInfo.Arguments} in {process.StartInfo.WorkingDirectory}"); + + process.Start(); + processOutputReader.BeginRead(); + + process.EnsureHighPriority(logger); + if (benchmarkCase.Job.Environment.HasValue(EnvironmentMode.AffinityCharacteristic)) + { + process.TrySetAffinity(benchmarkCase.Job.Environment.Affinity, logger); + } + + if (!process.WaitForExit(milliseconds: (int)TimeSpan.FromMinutes(10).TotalMilliseconds)) + { + logger.WriteLineInfo("// The benchmarking process did not finish within 10 minutes, it's going to get force killed now."); + + processOutputReader.CancelRead(); + consoleExitHandler.KillProcessTree(); + } + else + { + processOutputReader.StopRead(); + } + + ImmutableArray outputLines = processOutputReader.GetOutputLines(); + + return new ExecuteResult(true, + process.HasExited ? process.ExitCode : null, + process.Id, + outputLines.Where(line => !line.StartsWith("//")).ToArray(), + outputLines.Where(line => line.StartsWith("//")).ToArray(), + outputLines, + launchIndex); + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs index 3c7e75adac..326957d74a 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolChain.cs @@ -45,7 +45,7 @@ public static IToolchain From(NetCoreAppSettings netCoreAppSettings) // aot builds can be very slow logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm, retryFailedBuildWithNoDeps: false), - new Executor(), + new WasmExecutor(), netCoreAppSettings.CustomDotNetCliPath); } } diff --git a/src/BenchmarkDotNet/Toolchains/Results/BuildResult.cs b/src/BenchmarkDotNet/Toolchains/Results/BuildResult.cs index ac1a460b3c..cf2b5729e5 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/BuildResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/BuildResult.cs @@ -11,7 +11,7 @@ public class BuildResult : GenerateResult public string ErrorMessage { get; } private BuildResult(GenerateResult generateResult, bool isBuildSuccess, string errorMessage) - : base(generateResult.ArtifactsPaths, generateResult.IsGenerateSuccess, generateResult.GenerateException, generateResult.ArtifactsToCleanup, generateResult.NoAcknowledgments) + : base(generateResult.ArtifactsPaths, generateResult.IsGenerateSuccess, generateResult.GenerateException, generateResult.ArtifactsToCleanup) { IsBuildSuccess = isBuildSuccess; ErrorMessage = errorMessage; diff --git a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs index 194c40b262..769558ebeb 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs @@ -14,12 +14,25 @@ public class ExecuteResult public int? ExitCode { get; } public int? ProcessId { get; } public IReadOnlyList Errors => errors; - public IReadOnlyList Data => data; public IReadOnlyList Measurements => measurements; - public IReadOnlyList ExtraOutput { get; } + + /// + /// All lines printed to standard output by the Benchmark process + /// + public IReadOnlyList StandardOutput { get; } + + /// + /// Lines reported by the Benchmark process that are starting with "//" + /// + public IReadOnlyList PrefixedLines { get; } + + /// + /// Lines reported by the Benchmark process that are not starting with "//" + /// + public IReadOnlyList Results { get; } + internal readonly GcStats GcStats; internal readonly ThreadingStats ThreadingStats; - private readonly IReadOnlyList data; private readonly List errors; private readonly List measurements; @@ -27,15 +40,15 @@ public class ExecuteResult // that is why we search for Workload Results as they are produced at the end public bool IsSuccess => Measurements.Any(m => m.Is(IterationMode.Workload, IterationStage.Result)); - public ExecuteResult(bool foundExecutable, int? exitCode, int? processId, IReadOnlyList data, IReadOnlyList linesWithExtraOutput, int launchIndex) + public ExecuteResult(bool foundExecutable, int? exitCode, int? processId, IReadOnlyList results, IReadOnlyList prefixedLines, IReadOnlyList standardOutput, int launchIndex) { FoundExecutable = foundExecutable; - this.data = data; + Results = results; ProcessId = processId; ExitCode = exitCode; - ExtraOutput = linesWithExtraOutput; - - Parse(data, launchIndex, out measurements, out errors, out GcStats, out ThreadingStats); + PrefixedLines = prefixedLines; + StandardOutput = standardOutput; + Parse(results, prefixedLines, launchIndex, out measurements, out errors, out GcStats, out ThreadingStats); } internal ExecuteResult(List measurements, GcStats gcStats, ThreadingStats threadingStats) @@ -43,7 +56,7 @@ internal ExecuteResult(List measurements, GcStats gcStats, Threadin FoundExecutable = true; ExitCode = 0; errors = new List(); - ExtraOutput = Array.Empty(); + PrefixedLines = Array.Empty(); this.measurements = measurements; GcStats = gcStats; ThreadingStats = threadingStats; @@ -55,7 +68,7 @@ internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode : new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats); internal static ExecuteResult CreateFailed(int exitCode = -1) - => new ExecuteResult(false, exitCode, default, Array.Empty(), Array.Empty(), 0); + => new ExecuteResult(false, exitCode, default, Array.Empty(), Array.Empty(), Array.Empty(), 0); public override string ToString() => "ExecuteResult: " + (FoundExecutable ? "Found executable" : "Executable not found"); @@ -68,7 +81,7 @@ public void LogIssues(ILogger logger, BuildResult buildResult) // exit code can be different than 0 if the process has hanged at the end // so we check if some results were reported, if not then it was a failure - if (ExitCode != 0 && data.Count == 0) + if (ExitCode != 0 && Results.Count == 0) { logger.WriteLineError("ExitCode != 0 and no results reported"); } @@ -84,7 +97,7 @@ public void LogIssues(ILogger logger, BuildResult buildResult) } } - private static void Parse(IReadOnlyList data, int launchIndex, out List measurements, + private static void Parse(IReadOnlyList results, IReadOnlyList prefixedLines, int launchIndex, out List measurements, out List errors, out GcStats gcStats, out ThreadingStats threadingStats) { measurements = new List(); @@ -92,7 +105,16 @@ private static void Parse(IReadOnlyList data, int launchIndex, out List< gcStats = default; threadingStats = default; - foreach (string line in data.Where(text => !string.IsNullOrEmpty(text))) + foreach (string line in results.Where(text => !string.IsNullOrEmpty(text))) + { + Measurement measurement = Measurement.Parse(line, launchIndex); + if (measurement.IterationMode != IterationMode.Unknown) + { + measurements.Add(measurement); + } + } + + foreach (string line in prefixedLines.Where(text => !string.IsNullOrEmpty(text))) { if (line.StartsWith(ValidationErrorReporter.ConsoleErrorPrefix)) { @@ -106,14 +128,6 @@ private static void Parse(IReadOnlyList data, int launchIndex, out List< { threadingStats = ThreadingStats.Parse(line); } - else - { - Measurement measurement = Measurement.Parse(line, launchIndex); - if (measurement.IterationMode != IterationMode.Unknown) - { - measurements.Add(measurement); - } - } } if (errors.Count > 0) diff --git a/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs b/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs index 23d4affa06..b6de670b8d 100644 --- a/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs +++ b/src/BenchmarkDotNet/Toolchains/Results/GenerateResult.cs @@ -9,23 +9,21 @@ public class GenerateResult public bool IsGenerateSuccess { get; } public Exception GenerateException { get; } public IReadOnlyCollection ArtifactsToCleanup { get; } - public bool NoAcknowledgments { get; } public GenerateResult(ArtifactsPaths artifactsPaths, bool isGenerateSuccess, Exception generateException, - IReadOnlyCollection artifactsToCleanup, bool noAcknowledgments) + IReadOnlyCollection artifactsToCleanup) { ArtifactsPaths = artifactsPaths; IsGenerateSuccess = isGenerateSuccess; GenerateException = generateException; ArtifactsToCleanup = artifactsToCleanup; - NoAcknowledgments = noAcknowledgments; } - public static GenerateResult Success(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup, bool noAcknowledgments) - => new GenerateResult(artifactsPaths, true, null, artifactsToCleanup, noAcknowledgments); + public static GenerateResult Success(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup) + => new GenerateResult(artifactsPaths, true, null, artifactsToCleanup); public static GenerateResult Failure(ArtifactsPaths artifactsPaths, IReadOnlyCollection artifactsToCleanup, Exception exception = null) - => new GenerateResult(artifactsPaths, false, exception, artifactsToCleanup, false); + => new GenerateResult(artifactsPaths, false, exception, artifactsToCleanup); public override string ToString() => "GenerateResult: " + (IsGenerateSuccess ? "Success" : "Fail"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests.FSharp/Program.fs b/tests/BenchmarkDotNet.IntegrationTests.FSharp/Program.fs index b90d780ce5..e8192e4c62 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.FSharp/Program.fs +++ b/tests/BenchmarkDotNet.IntegrationTests.FSharp/Program.fs @@ -28,13 +28,9 @@ type Db() = type TestEnum = | A = 0 | B = 1 | C = 2 type EnumParamsTest() = - let mutable collectedParams = HashSet() - - [] + [] member val EnumParamValue = TestEnum.A with get, set [] member this.Benchmark() = - if not <| collectedParams.Contains(this.EnumParamValue) then - printfn "// ### New Parameter %A ###" this.EnumParamValue - collectedParams.Add(this.EnumParamValue) |> ignore \ No newline at end of file + if not (this.EnumParamValue = TestEnum.B) then failwith "Invalid Params value assigned" diff --git a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs index dc4cf9bfee..6975a01401 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AllSetupAndCleanupTest.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using Xunit; @@ -45,16 +46,18 @@ public class AllSetupAndCleanupTest : BenchmarkTestExecutor public AllSetupAndCleanupTest(ITestOutputHelper output) : base(output) { } + private static string[] GetActualLogLines(Summary summary) + => GetSingleStandardOutput(summary).Where(line => line.StartsWith(Prefix)).ToArray(); + [Fact] public void AllSetupAndCleanupMethodRunsTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); @@ -84,13 +87,12 @@ public class AllSetupAndCleanupAttributeBenchmarks [Fact] public void AllSetupAndCleanupMethodRunsAsyncTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); @@ -120,13 +122,12 @@ public class AllSetupAndCleanupAttributeBenchmarksAsync [Fact] public void AllSetupAndCleanupMethodRunsAsyncTaskSetupTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); @@ -156,13 +157,12 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncTaskSetup [Fact] public void AllSetupAndCleanupMethodRunsAsyncGenericTaskSetupTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); @@ -202,13 +202,12 @@ public async Task GlobalCleanup() [Fact] public void AllSetupAndCleanupMethodRunsAsyncValueTaskSetupTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); @@ -238,13 +237,12 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup [FactNotGitHubActionsWindows] public void AllSetupAndCleanupMethodRunsAsyncGenericValueTaskSetupTest() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); - var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); + var actualLogLines = GetActualLogLines(summary); foreach (string line in actualLogLines) Output.WriteLine(line); Assert.Equal(expectedLogLines, actualLogLines); diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs index a5e3060315..64d270b3d6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs @@ -8,6 +8,8 @@ using BenchmarkDotNet.Tests.Loggers; using Xunit; using Xunit.Abstractions; +using System.Collections.Generic; +using BenchmarkDotNet.Reports; namespace BenchmarkDotNet.IntegrationTests { @@ -91,5 +93,11 @@ protected IConfig CreateSimpleConfig(OutputLogger logger = null, Job job = null) .AddColumnProvider(DefaultColumnProviders.Instance) .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()); } + + protected static IReadOnlyList GetSingleStandardOutput(Summary summary) + => summary.Reports.Single().ExecuteResults.Single().StandardOutput; + + protected static IReadOnlyList GetCombinedStandardOutput(Summary summary) + => summary.Reports.SelectMany(r => r.ExecuteResults).SelectMany(e => e.StandardOutput).ToArray(); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index c1b4c75837..5efd67a67e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -31,7 +31,8 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() .WithToolchain(NativeAotToolchain.CreateBuilder() .UseNuGet( "6.0.0-rc.1.21420.1", // we test against specific version to keep this test stable - "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json") // using old feed that supports net5.0 + "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json") // using old feed that supports net6.0 + .TargetFrameworkMoniker("net6.0") .ToToolchain())); var summary = CanExecute(config, fullValidation: false); diff --git a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs index b1e268673c..29707c917f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs @@ -1,5 +1,4 @@ using System; -using System.Linq; using System.Collections.Generic; using BenchmarkDotNet.Attributes; using Xunit; @@ -31,14 +30,11 @@ public void CustomEnginesAreSupported() var summary = CanExecute(config, fullValidation: false); - AssertMessageGotDisplayed(summary, GlobalSetupMessage); - AssertMessageGotDisplayed(summary, EngineRunMessage); - AssertMessageGotDisplayed(summary, GlobalCleanupMessage); - } + IReadOnlyList standardOutput = GetSingleStandardOutput(summary); - private static void AssertMessageGotDisplayed(Summary summary, string message) - { - Assert.True(summary.Reports.Any(report => report.ExecuteResults.Any(executeResult => executeResult.ExtraOutput.Any(line => line == message))), $"{message} should have been printed by custom Engine"); + Assert.Contains(GlobalSetupMessage, standardOutput); + Assert.Contains(EngineRunMessage, standardOutput); + Assert.Contains(GlobalCleanupMessage, standardOutput); } public class SimpleBenchmark diff --git a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs index 5a6b6eb76e..b9ca511236 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DryRunTests.cs @@ -1,7 +1,8 @@ using System; +using System.Linq; using System.Threading; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Engines; using Xunit; using Xunit.Abstractions; @@ -12,7 +13,7 @@ public class DryRunTests : BenchmarkTestExecutor public DryRunTests(ITestOutputHelper output) : base(output) { } [Fact] - public void WelchTTest() => CanExecute(CreateSimpleConfig()); + public void WelchTTest() => CanExecute(); [DryJob, StatisticalTestColumn] public class WelchTTestBench @@ -24,19 +25,22 @@ public class WelchTTestBench public void B() => Thread.Sleep(10); } - private const string CounterPrefix = "// Counter = "; - [Fact] public void ColdStart() { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); + var summary = CanExecute(); + + var report = summary.Reports.Single(); - CanExecute(config); + Assert.Equal(2, report.AllMeasurements.Count); - string log = logger.GetLog(); - Assert.Contains($"{CounterPrefix}1", log); - Assert.DoesNotContain($"{CounterPrefix}2", log); + foreach (var measurement in report.AllMeasurements) + { + Assert.Equal(1, measurement.LaunchIndex); + Assert.Equal(1, measurement.IterationIndex); + Assert.Equal(IterationMode.Workload, measurement.IterationMode); + Assert.True(measurement.IterationStage is IterationStage.Actual or IterationStage.Result); + } } [DryJob] @@ -47,8 +51,10 @@ public class ColdStartBench [Benchmark] public void Foo() { - counter++; - Console.WriteLine($"{CounterPrefix}{counter}"); + if (++counter > 1) + { + throw new InvalidOperationException("Benchmark was executed more than once"); + } } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/FSharpTests.cs b/tests/BenchmarkDotNet.IntegrationTests/FSharpTests.cs index 89bd3e29ca..2b0928e9b4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/FSharpTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/FSharpTests.cs @@ -1,8 +1,5 @@ -using System; -using Xunit; +using Xunit; using Xunit.Abstractions; - -using BenchmarkDotNet.Tests.Loggers; using static FSharpBenchmarks; namespace BenchmarkDotNet.IntegrationTests @@ -12,15 +9,6 @@ public class FSharpTests : BenchmarkTestExecutor public FSharpTests(ITestOutputHelper output) : base(output) { } [Fact] - public void ParamsSupportFSharpEnums() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - foreach (var param in new[] { TestEnum.A, TestEnum.B }) - Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); - Assert.DoesNotContain($"// ### New Parameter {TestEnum.C} ###" + Environment.NewLine, logger.GetLog()); - } + public void ParamsSupportFSharpEnums() => CanExecute(); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/GenericBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/GenericBenchmarkTest.cs deleted file mode 100644 index d5553bd5d6..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/GenericBenchmarkTest.cs +++ /dev/null @@ -1,96 +0,0 @@ -using System; -using System.Linq; -using System.Reflection; -using System.Threading; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class GenericBenchmarkTest : BenchmarkTestExecutor - { - public GenericBenchmarkTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void GenericClassesAreSupported() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - var expected1 = $"// ### Benchmark: SerializationLibrary1, Type: {typeof(FlatClassBenchmark).Name} ###"; - Assert.Contains(expected1, logger.GetLog()); - - logger.ClearLog(); - CanExecute(config); - var expected2 = $"// ### Benchmark: SerializationLibrary2, Type: {typeof(DoubleArrayBenchmark).Name} ###"; - Assert.Contains(expected2, logger.GetLog()); - } - } - - // From https://github.com/dotnet/BenchmarkDotNet/issues/44 - public abstract class AbstractBenchmark - { - private readonly T value; - private readonly Serializer1 serializer1 = new Serializer1(); - private readonly Serializer2 serializer2 = new Serializer2(); - - protected AbstractBenchmark() - { - value = CreateValue(); - } - - protected abstract T CreateValue(); - - [Benchmark] - public void SerializationLibrary1() - { - Thread.Sleep(10); - // Our direct type is BenchmarkDotNet.Autogenerated.Program, - // which inherits from BenchmarkDotNet.IntegrationTests.FlatClassBenchmark - Console.WriteLine($"// ### Benchmark: SerializationLibrary1, Type: {GetType().GetTypeInfo().BaseType.Name} ###"); - string text = serializer1.Serialize(value); - } - - [Benchmark] - public void SerializationLibrary2() - { - Thread.Sleep(10); - // Our direct type is BenchmarkDotNet.Autogenerated.Program, - // which inherits from BenchmarkDotNet.IntegrationTests.FlatClassBenchmark - Console.WriteLine($"// ### Benchmark: SerializationLibrary2, Type: {GetType().GetTypeInfo().BaseType.Name} ###"); - string text = serializer2.Serialize(value); - } - } - - [Config(typeof(SingleRunFastConfig))] - public class FlatClassBenchmark : AbstractBenchmark - { - protected override FlatClass CreateValue() => new FlatClass() { Number = 42, Text = "64", TimeStamp = DateTime.UtcNow }; - } - - [Config(typeof(SingleRunFastConfig))] - public class DoubleArrayBenchmark : AbstractBenchmark - { - protected override double[] CreateValue() => Enumerable.Repeat(42.0, 100 * 1000).ToArray(); - } - - public class FlatClass - { - public int Number { get; set; } - public string Text { get; set; } - public DateTime TimeStamp { get; set; } - } - - public class Serializer1 - { - public string Serialize(T value) => ""; - } - - public class Serializer2 - { - public string Serialize(T value) => ""; - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTargetTest.cs b/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTargetTest.cs deleted file mode 100644 index e3060151e1..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTargetTest.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class GlobalCleanupAttributeTargetTest : BenchmarkTestExecutor - { - private const string BaselineGlobalCleanupCalled = "// ### Baseline GlobalCleanup called ###"; - private const string BaselineBenchmarkCalled = "// ### Baseline Benchmark called ###"; - private const string FirstGlobalCleanupCalled = "// ### First GlobalCleanup called ###"; - private const string FirstBenchmarkCalled = "// ### First Benchmark called ###"; - private const string SecondGlobalCleanupCalled = "// ### Second GlobalCleanup called ###"; - private const string SecondBenchmarkCalled = "// ### Second Benchmark called ###"; - - public GlobalCleanupAttributeTargetTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void GlobalCleanupTargetSpecificMethodTest() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - string log = logger.GetLog(); - - Assert.Contains(BaselineBenchmarkCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(BaselineBenchmarkCalled + Environment.NewLine) < - log.IndexOf(BaselineGlobalCleanupCalled + Environment.NewLine)); - - Assert.Contains(FirstGlobalCleanupCalled + Environment.NewLine, log); - Assert.Contains(FirstBenchmarkCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(FirstBenchmarkCalled + Environment.NewLine) < - log.IndexOf(FirstGlobalCleanupCalled + Environment.NewLine)); - - Assert.Contains(SecondGlobalCleanupCalled + Environment.NewLine, log); - Assert.Contains(SecondBenchmarkCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(SecondBenchmarkCalled + Environment.NewLine) < - log.IndexOf(SecondGlobalCleanupCalled + Environment.NewLine)); - } - - public class GlobalCleanupAttributeTargetBenchmarks - { - private int cleanupValue; - - [Benchmark(Baseline = true)] - public void BaselineBenchmark() - { - cleanupValue = -1; - - Console.WriteLine(BaselineBenchmarkCalled); - } - - [GlobalCleanup] - public void BaselineCleanup() - { - Assert.Equal(-1, cleanupValue); - - Console.WriteLine(BaselineGlobalCleanupCalled); - } - - [Benchmark] - public void Benchmark1() - { - cleanupValue = 1; - - Console.WriteLine(FirstBenchmarkCalled); - } - - [GlobalCleanup(Target = nameof(Benchmark1))] - public void GlobalCleanup1() - { - Assert.Equal(1, cleanupValue); - - Console.WriteLine(FirstGlobalCleanupCalled); - } - - [Benchmark] - public void Benchmark2() - { - cleanupValue = 2; - - Console.WriteLine(SecondBenchmarkCalled); - } - - [GlobalCleanup(Target = nameof(Benchmark2))] - public void GlobalCleanup2() - { - Assert.Equal(2, cleanupValue); - - Console.WriteLine(SecondGlobalCleanupCalled); - } - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTest.cs b/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTest.cs deleted file mode 100644 index ee303f5120..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/GlobalCleanupAttributeTest.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class GlobalCleanupAttributeTest : BenchmarkTestExecutor - { - private const string GlobalCleanupCalled = "// ### GlobalCleanup called ###"; - private const string BenchmarkCalled = "// ### Benchmark called ###"; - - public GlobalCleanupAttributeTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void GlobalCleanupMethodRunsTest() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - string log = logger.GetLog(); - Assert.Contains(BenchmarkCalled + System.Environment.NewLine, log); - Assert.True( - log.IndexOf(BenchmarkCalled + System.Environment.NewLine) < - log.IndexOf(GlobalCleanupCalled + System.Environment.NewLine)); - } - - public class GlobalCleanupAttributeBenchmarks - { - [Benchmark] - public void Benchmark() - { - Console.WriteLine(BenchmarkCalled); - } - - [GlobalCleanup] - public void GlobalCleanup() - { - Console.WriteLine(GlobalCleanupCalled); - } - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTargetTest.cs b/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTargetTest.cs deleted file mode 100644 index 20431f53b9..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTargetTest.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class GlobalSetupAttributeTargetTest : BenchmarkTestExecutor - { - private const string BaselineGlobalSetupCalled = "// ### Baseline GlobalSetup called ###"; - private const string BaselineBenchmarkCalled = "// ### Baseline Benchmark called ###"; - private const string FirstGlobalSetupCalled = "// ### First GlobalSetup called ###"; - private const string FirstBenchmarkCalled = "// ### First Benchmark called ###"; - private const string SecondGlobalSetupCalled = "// ### Second GlobalSetup called ###"; - private const string SecondBenchmarkCalled = "// ### Second Benchmark called ###"; - - public GlobalSetupAttributeTargetTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void GlobalSetupTargetSpecificMethodTest() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - string log = logger.GetLog(); - - Assert.Contains(BaselineGlobalSetupCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(BaselineGlobalSetupCalled + Environment.NewLine) < - log.IndexOf(BaselineBenchmarkCalled + Environment.NewLine)); - - Assert.Contains(FirstGlobalSetupCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(FirstGlobalSetupCalled + Environment.NewLine) < - log.IndexOf(FirstBenchmarkCalled + Environment.NewLine)); - - Assert.Contains(SecondGlobalSetupCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(SecondGlobalSetupCalled + Environment.NewLine) < - log.IndexOf(SecondBenchmarkCalled + Environment.NewLine)); - } - - public class GlobalSetupAttributeTargetBenchmarks - { - private int setupValue; - - [GlobalSetup] - public void BaselineSetup() - { - setupValue = -1; - - Console.WriteLine(BaselineGlobalSetupCalled); - } - - [Benchmark(Baseline = true)] - public void BaselineBenchmark() - { - Assert.Equal(-1, setupValue); - - Console.WriteLine(BaselineBenchmarkCalled); - } - - [GlobalSetup(Target = nameof(Benchmark1))] - public void GlobalSetup1() - { - setupValue = 1; - - Console.WriteLine(FirstGlobalSetupCalled); - } - - [Benchmark] - public void Benchmark1() - { - Assert.Equal(1, setupValue); - - Console.WriteLine(FirstBenchmarkCalled); - } - - [GlobalSetup(Target = nameof(Benchmark2))] - public void GlobalSetup2() - { - setupValue = 2; - - Console.WriteLine(SecondGlobalSetupCalled); - } - - [Benchmark] - public void Benchmark2() - { - Assert.Equal(2, setupValue); - - Console.WriteLine(SecondBenchmarkCalled); - } - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTest.cs b/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTest.cs deleted file mode 100644 index 7f1cf0e76b..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/GlobalSetupAttributeTest.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class GlobalSetupAttributeTest : BenchmarkTestExecutor - { - private const string GlobalSetupCalled = "// ### GlobalSetup called ###"; - private const string BenchmarkCalled = "// ### Benchmark called ###"; - - public GlobalSetupAttributeTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void GlobalSetupMethodRunsTest() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - string log = logger.GetLog(); - Assert.Contains(GlobalSetupCalled + Environment.NewLine, log); - Assert.True( - log.IndexOf(GlobalSetupCalled + Environment.NewLine) < - log.IndexOf(BenchmarkCalled + Environment.NewLine)); - } - - public class GlobalSetupAttributeBenchmarks - { - [GlobalSetup] - public void GlobalSetup() - { - Console.WriteLine(GlobalSetupCalled); - } - - [Benchmark] - public void Benchmark() - { - Console.WriteLine(BenchmarkCalled); - } - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/InnerClassTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InnerClassTest.cs deleted file mode 100644 index 08b4fa950b..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/InnerClassTest.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - // See https://github.com/dotnet/BenchmarkDotNet/issues/55 - // https://github.com/dotnet/BenchmarkDotNet/issues/59 is also related - public class InnerClassTest : BenchmarkTestExecutor - { - public InnerClassTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void InnerClassesAreSupported() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - var testLog = logger.GetLog(); - Assert.Contains("// ### BenchmarkInnerClass method called ###" + Environment.NewLine, testLog); - Assert.Contains("// ### BenchmarkGenericInnerClass method called ###" + Environment.NewLine, testLog); - Assert.DoesNotContain("No benchmarks found", logger.GetLog()); - } - - public class Inner - { - [Benchmark] - public Tuple BenchmarkInnerClass() - { - Console.WriteLine("// ### BenchmarkInnerClass method called ###"); - return Tuple.Create(new Outer(), new Outer.Inner()); - } - - [Benchmark] - public Tuple> BenchmarkGenericInnerClass() - { - Console.WriteLine("// ### BenchmarkGenericInnerClass method called ###"); - return Tuple.Create(new Outer(), new Outer.InnerGeneric()); - } - } - } - - public class Outer - { - public class Inner - { - } - - public class InnerGeneric - { - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs index a87e43997b..92b6b85106 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs @@ -1,10 +1,9 @@ -using System; +using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using Xunit; @@ -12,79 +11,63 @@ namespace BenchmarkDotNet.IntegrationTests { - public class JitRuntimeValidationTest + public class JitRuntimeValidationTest : BenchmarkTestExecutor { - protected readonly ITestOutputHelper Output; + public JitRuntimeValidationTest(ITestOutputHelper output) : base(output) { } - private class PlatformConfig : ManualConfig - { - public PlatformConfig(Runtime runtime, Jit jit, Platform platform) - { - AddJob(new Job(Job.Dry, new EnvironmentMode() - { - Runtime = runtime, - Jit = jit, - Platform = platform - })); - } - } - - private const string OkCaption = "// OkCaption"; - // private const string LegacyJitNotAvailableForMono = "// ERROR: LegacyJIT is requested but it is not available for Mono"; +// private const string LegacyJitNotAvailableForMono = "// ERROR: LegacyJIT is requested but it is not available for Mono"; private const string RyuJitNotAvailable = "// ERROR: RyuJIT is requested but it is not available in current environment"; private const string ToolchainSupportsOnlyRyuJit = "Currently dotnet cli toolchain supports only RyuJit"; - public JitRuntimeValidationTest(ITestOutputHelper outputHelper) - { - Output = outputHelper; - } - [TheoryWindowsOnly("CLR is a valid job only on Windows")] - [InlineData(Jit.LegacyJit, Platform.X86, OkCaption)] - [InlineData(Jit.LegacyJit, Platform.X64, OkCaption)] + [InlineData(Jit.LegacyJit, Platform.X86, null)] + [InlineData(Jit.LegacyJit, Platform.X64, null)] [InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)] - [InlineData(Jit.RyuJit, Platform.X64, OkCaption)] - public void CheckClrOnWindows(Jit jit, Platform platform, string expectedText) + [InlineData(Jit.RyuJit, Platform.X64, null)] + public void CheckClrOnWindows(Jit jit, Platform platform, string errorMessage) { - Verify(ClrRuntime.Net462, jit, platform, expectedText); + Verify(ClrRuntime.Net462, jit, platform, errorMessage); } -// [TheoryWindowsOnly("CLR is a valid job only on Windows")] -// [InlineData(Jit.LegacyJit, Platform.X86, LegacyJitNotAvailableForMono)] -// [InlineData(Jit.LegacyJit, Platform.X64, LegacyJitNotAvailableForMono)] -// [InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)] -// [InlineData(Jit.RyuJit, Platform.X64, RyuJitNotAvailable)] -// public void CheckMono(Jit jit, Platform platform, string expectedText) -// { -// Verify(Runtime.Mono, jit, platform, expectedText); -// } +// [TheoryWindowsOnly("CLR is a valid job only on Windows")] +// [InlineData(Jit.LegacyJit, Platform.X86, LegacyJitNotAvailableForMono)] +// [InlineData(Jit.LegacyJit, Platform.X64, LegacyJitNotAvailableForMono)] +// [InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)] +// [InlineData(Jit.RyuJit, Platform.X64, RyuJitNotAvailable)] +// public void CheckMono(Jit jit, Platform platform, string errorMessage) +// { +// Verify(Runtime.Mono, jit, platform, errorMessage); +// } [Theory] [InlineData(Jit.LegacyJit, Platform.X86, ToolchainSupportsOnlyRyuJit)] [InlineData(Jit.LegacyJit, Platform.X64, ToolchainSupportsOnlyRyuJit)] - [InlineData(Jit.RyuJit, Platform.X64, OkCaption)] - public void CheckCore(Jit jit, Platform platform, string expectedText) + [InlineData(Jit.RyuJit, Platform.X64, null)] + public void CheckCore(Jit jit, Platform platform, string errorMessage) { - Verify(CoreRuntime.Core60, jit, platform, expectedText); + Verify(CoreRuntime.Core60, jit, platform, errorMessage); } - private void Verify(Runtime runtime, Jit jit, Platform platform, string expectedText) + private void Verify(Runtime runtime, Jit jit, Platform platform, string errorMessage) { var logger = new OutputLogger(Output); - var config = new PlatformConfig(runtime, jit, platform).AddLogger(logger).AddColumnProvider(DefaultColumnProviders.Instance); + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithPlatform(platform).WithJit(jit).WithRuntime(runtime)) + .AddLogger(logger) + .AddColumnProvider(DefaultColumnProviders.Instance); - BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(TestBenchmark), config) }); + CanExecute(config, fullValidation: errorMessage is null); - Assert.Contains(expectedText, logger.GetLog()); + if (errorMessage is not null) + { + Assert.Contains(errorMessage, logger.GetLog()); + } } public class TestBenchmark { [Benchmark] - public void Benchmark() - { - Console.WriteLine(OkCaption); - } + public void Benchmark() { } } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsAttributeStaticTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsAttributeStaticTest.cs deleted file mode 100644 index 189a42137c..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsAttributeStaticTest.cs +++ /dev/null @@ -1,148 +0,0 @@ -using System; -using System.Collections.Generic; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class ParamsTestStaticPropertyTest : BenchmarkTestExecutor - { - public ParamsTestStaticPropertyTest(ITestOutputHelper output) : base(output) { } - - [Fact] - public void Test() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - foreach (var param in new[] { 1, 2 }) - Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); - Assert.DoesNotContain($"// ### New Parameter {default(int)} ###" + Environment.NewLine, logger.GetLog()); - } - - public class ParamsTestStaticProperty - { - /// - /// Deliberately made the Property "static" to ensure that Params also work okay in this scenario - /// - [Params(1, 2)] - public static int StaticParamProperty { get; set; } - - private static HashSet collectedParams = new HashSet(); - - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(StaticParamProperty) == false) - { - Console.WriteLine($"// ### New Parameter {StaticParamProperty} ###"); - collectedParams.Add(StaticParamProperty); - } - } - } - } - - public class ParamsTestStaticPrivatePropertyError : BenchmarkTestExecutor - { - public ParamsTestStaticPrivatePropertyError(ITestOutputHelper output) : base(output) { } - - /// - /// Deliberately made the Property "static" to ensure that Params also work okay in this scenario - /// - [Params(1, 2)] - public static int StaticParamProperty { get; private set; } - - private static HashSet collectedParams = new HashSet(); - - [Fact] - public void Test() - { - // System.InvalidOperationException : Property "StaticParamProperty" must be public and writable if it has the [Params(..)] attribute applied to it - Assert.Throws(() => CanExecute()); - } - -#pragma warning disable xUnit1013 // Public method should be marked as test - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(StaticParamProperty) == false) - { - Console.WriteLine($"// ### New Parameter {StaticParamProperty} ###"); - collectedParams.Add(StaticParamProperty); - } - } -#pragma warning restore xUnit1013 // Public method should be marked as test - } - - // Deliberately made everything "static" (as well as using a Field) to ensure that Params also work okay in this scenario - public class ParamsTestStaticFieldTest : BenchmarkTestExecutor - { - public ParamsTestStaticFieldTest(ITestOutputHelper output) : base(output) - { - } - - [Fact] - public void Test() - { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); - - foreach (var param in new[] { 1, 2 }) - Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); - Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog()); - } - - public class ParamsTestStaticField - { - [Params(1, 2)] - public static int StaticParamField = 0; - - private static HashSet collectedParams = new HashSet(); - - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(StaticParamField) == false) - { - Console.WriteLine($"// ### New Parameter {StaticParamField} ###"); - collectedParams.Add(StaticParamField); - } - } - } - } - - // Deliberately made everything "static" (as well as using a Field) to ensure that Params also work okay in this scenario - [Config(typeof(SingleRunFastConfig))] - public class ParamsTestStaticPrivateFieldError - { - [Params(1, 2)] - private static int StaticParamField = 0; - - private static HashSet collectedParams = new HashSet(); - - [Fact] - public static void Test() - { - // System.InvalidOperationException : Field "StaticParamField" must be public if it has the [Params(..)] attribute applied to it - Assert.Throws(() => new BenchmarkTestExecutor().CanExecute()); - } - -#pragma warning disable xUnit1013 // Public method should be marked as test - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(StaticParamField) == false) - { - Console.WriteLine($"// ### New Parameter {StaticParamField} ###"); - collectedParams.Add(StaticParamField); - } - } -#pragma warning restore xUnit1013 // Public method should be marked as test - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs index 23a4a52222..d610793bf1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ParamsTests.cs @@ -1,7 +1,5 @@ using System; -using System.Collections.Generic; using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Tests.Loggers; using Xunit; using Xunit.Abstractions; @@ -14,13 +12,13 @@ public ParamsTests(ITestOutputHelper output) : base(output) { } [Fact] public void ParamsSupportPropertyWithPublicSetter() { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); + var summary = CanExecute(); + var standardOutput = GetCombinedStandardOutput(summary); - CanExecute(config); foreach (var param in new[] { 1, 2 }) - Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); - Assert.DoesNotContain($"// ### New Parameter {default(int)} ###" + Environment.NewLine, logger.GetLog()); + Assert.Contains($"// ### New Parameter {param} ###", standardOutput); + + Assert.DoesNotContain($"// ### New Parameter {default(int)} ###", standardOutput); } public class ParamsTestProperty @@ -28,17 +26,8 @@ public class ParamsTestProperty [Params(1, 2)] public int ParamProperty { get; set; } - private HashSet collectedParams = new HashSet(); - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(ParamProperty) == false) - { - Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); - collectedParams.Add(ParamProperty); - } - } + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); } [Fact] @@ -53,30 +42,20 @@ public class ParamsTestPrivatePropertyError [Params(1, 2)] public int ParamProperty { get; private set; } - private HashSet collectedParams = new HashSet(); - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(ParamProperty) == false) - { - Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); - collectedParams.Add(ParamProperty); - } - } + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamProperty} ###"); } [Fact] public void ParamsSupportPublicFields() { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); - - CanExecute(config); + var summary = CanExecute(); + var standardOutput = GetCombinedStandardOutput(summary); foreach (var param in new[] { 1, 2 }) - Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); - Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog()); + Assert.Contains($"// ### New Parameter {param} ###", standardOutput); + + Assert.DoesNotContain($"// ### New Parameter 0 ###", standardOutput); } public class ParamsTestField @@ -84,17 +63,8 @@ public class ParamsTestField [Params(1, 2)] public int ParamField = 0; - private HashSet collectedParams = new HashSet(); - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(ParamField) == false) - { - Console.WriteLine($"// ### New Parameter {ParamField} ###"); - collectedParams.Add(ParamField); - } - } + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamField} ###"); } [Fact] @@ -109,17 +79,8 @@ public class ParamsTestPrivateFieldError [Params(1, 2)] private int ParamField = 0; - private HashSet collectedParams = new HashSet(); - [Benchmark] - public void Benchmark() - { - if (collectedParams.Contains(ParamField) == false) - { - Console.WriteLine($"// ### New Parameter {ParamField} ###"); - collectedParams.Add(ParamField); - } - } + public void Benchmark() => Console.WriteLine($"// ### New Parameter {ParamField} ###"); } public enum NestedOne @@ -216,5 +177,52 @@ public void AcceptingArray() throw new InvalidOperationException($"Incorrect array element at index {i}, was {Array[i]} instead of {i}"); } } + + [Fact] + public void StaticFieldsAndPropertiesCanBeParams() => CanExecute(); + + public class WithStaticParams + { + [Params(1)] + public static int StaticParamField = 0; + + [Params(2)] + public static int StaticParamProperty { get; set; } = 0; + + [Benchmark] + public void Test() + { + if (StaticParamField != 1) + throw new ArgumentException($"{nameof(StaticParamField)} has wrong value: {StaticParamField}!"); + if (StaticParamProperty != 2) + throw new ArgumentException($"{nameof(StaticParamProperty)} has wrong value: {StaticParamProperty}!"); + } + } + + [Fact] + public void ParamsPropertiesMustHavePublicSetter() + => Assert.Throws(() => CanExecute()); + + public class WithStaticParamsPropertyWithNoPublicSetter + { + [Params(3)] + public static int StaticParamProperty { get; private set; } + + [Benchmark] + public int Benchmark() => StaticParamProperty; + } + + [Fact] + public void ParamsFieldsMustBePublic() + => Assert.Throws(() => CanExecute()); + + public class WithStaticPrivateParamsField + { + [Params(4)] + private static int StaticParamField = 0; + + [Benchmark] + public int Benchmark() => StaticParamField; + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/PerformanceUnitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/PerformanceUnitTest.cs deleted file mode 100644 index 73224a087f..0000000000 --- a/tests/BenchmarkDotNet.IntegrationTests/PerformanceUnitTest.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.Linq; -using System.Threading; -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Tests.Loggers; -using Xunit; -using Xunit.Abstractions; - -namespace BenchmarkDotNet.IntegrationTests -{ - public class PerformanceUnitTestRunner - { - private readonly ITestOutputHelper output; - - public PerformanceUnitTestRunner(ITestOutputHelper outputHelper) - { - output = outputHelper; - } - - // See also: https://github.com/dotnet/BenchmarkDotNet/issues/204 - [Fact(Skip = "This test fails on AppVeyor")] - public void Test() - { - var logger = new OutputLogger(output); - var config = DefaultConfig.Instance.AddLogger(logger); - var summary = BenchmarkRunner.Run(config); - - // Sanity checks, to be sure that the different benchmarks actually run - var testOutput = logger.GetLog(); - Assert.Contains("// ### Slow Benchmark called ###" + Environment.NewLine, testOutput); - Assert.Contains("// ### Fast Benchmark called ###" + Environment.NewLine, testOutput); - - // Check that slow benchmark is actually slower than the fast benchmark! - var slowBenchmarkRun = summary.GetRunsFor(r => r.SlowBenchmark()).First(); - var fastBenchmarkRun = summary.GetRunsFor(r => r.FastBenchmark()).First(); - Assert.True(slowBenchmarkRun.GetAverageTime() > fastBenchmarkRun.GetAverageTime(), - string.Format("Expected SlowBenchmark: {0:N2} ns to be MORE than FastBenchmark: {1:N2} ns", - slowBenchmarkRun.GetAverageTime().Nanoseconds, fastBenchmarkRun.GetAverageTime().Nanoseconds)); - Assert.True(slowBenchmarkRun.GetOpsPerSecond() < fastBenchmarkRun.GetOpsPerSecond(), - string.Format("Expected SlowBenchmark: {0:N2} Ops to be LESS than FastBenchmark: {1:N2} Ops", - slowBenchmarkRun.GetOpsPerSecond(), fastBenchmarkRun.GetOpsPerSecond())); - - // Whilst we're at it, let's do more specific Asserts as we know what the elapsed time should be - var slowBenchmarkReport = summary.GetReportFor(r => r.SlowBenchmark()); - var fastBenchmarkReport = summary.GetReportFor(r => r.FastBenchmark()); - foreach (var slowRun in slowBenchmarkReport.GetResultRuns()) - Assert.InRange(slowRun.GetAverageTime().Nanoseconds / 1000.0 / 1000.0, low: 98, high: 102); - foreach (var fastRun in fastBenchmarkReport.GetResultRuns()) - Assert.InRange(fastRun.GetAverageTime().Nanoseconds / 1000.0 / 1000.0, low: 14, high: 17); - } - } - - [Config(typeof(SingleRunFastConfig))] - public class PerformanceUnitTest - { - [Benchmark] - public void FastBenchmark() - { - Console.WriteLine("// ### Fast Benchmark called ###"); - Thread.Sleep(15); - } - - [Benchmark] - public void SlowBenchmark() - { - Console.WriteLine("// ### Slow Benchmark called ###"); - Thread.Sleep(100); - } - } -} diff --git a/tests/BenchmarkDotNet.IntegrationTests/PowerManagementApplierTests.cs b/tests/BenchmarkDotNet.IntegrationTests/PowerManagementApplierTests.cs index de129c9ac1..46bc67d655 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/PowerManagementApplierTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/PowerManagementApplierTests.cs @@ -18,8 +18,7 @@ public PowerManagementApplierTests(ITestOutputHelper output) : base(output) { } public void TestSettingAndRevertingBackGuid() { var userPlan = PowerManagementHelper.CurrentPlan; - var logger = new OutputLogger(Output); - var powerManagementApplier = new PowerManagementApplier(logger); + var powerManagementApplier = new PowerManagementApplier(new OutputLogger(Output)); powerManagementApplier.ApplyPerformancePlan(PowerManagementApplier.Map(PowerPlan.HighPerformance)); @@ -34,8 +33,7 @@ public void TestSettingAndRevertingBackGuid() public void TestPowerPlanShouldNotChange() { var userPlan = PowerManagementHelper.CurrentPlan; - var logger = new OutputLogger(Output); - var powerManagementApplier = new PowerManagementApplier(logger); + var powerManagementApplier = new PowerManagementApplier(new OutputLogger(Output)); powerManagementApplier.ApplyPerformancePlan(PowerManagementApplier.Map(PowerPlan.UserPowerPlan)); diff --git a/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs b/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs index 4976362314..47ba0b4089 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/PriorityTests.cs @@ -14,8 +14,7 @@ public PriorityTests(ITestOutputHelper output) : base(output) { } [Fact] public void ParamsSupportPropertyWithPublicSetter() { - var logger = new OutputLogger(Output); - var config = CreateSimpleConfig(logger); + var config = CreateSimpleConfig(); var summary = CanExecute(config); var columns = summary.Table.Columns; diff --git a/tests/BenchmarkDotNet.IntegrationTests/ProcessorArchitectureTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ProcessorArchitectureTest.cs index a763a9b9a8..ea942ef8f5 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ProcessorArchitectureTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ProcessorArchitectureTest.cs @@ -11,12 +11,6 @@ namespace BenchmarkDotNet.IntegrationTests { public class ProcessorArchitectureTest : BenchmarkTestExecutor { - private const string X86FailedCaption = "// x86FAILED"; - private const string X64FailedCaption = "// x64FAILED"; - private const string AnyCpuOkCaption = "// AnyCpuOkCaption"; - private const string HostPlatformOkCaption = "// HostPlatformOkCaption"; - private const string BenchmarkNotFound = "// There are no benchmarks found"; - public ProcessorArchitectureTest(ITestOutputHelper outputHelper) : base(outputHelper) { } @@ -25,25 +19,20 @@ public ProcessorArchitectureTest(ITestOutputHelper outputHelper) : base(outputHe public void SpecifiedProcessorArchitectureMustBeRespected() { #if NETFRAMEWORK // dotnet cli does not support x86 compilation so far, so I disable this test - Verify(Platform.X86, typeof(X86Benchmark), X86FailedCaption); + Verify(Platform.X86, typeof(X86Benchmark)); #endif - Verify(Platform.X64, typeof(X64Benchmark), X64FailedCaption); - Verify(Platform.AnyCpu, typeof(AnyCpuBenchmark), "nvm"); + Verify(Platform.X64, typeof(X64Benchmark)); + Verify(Platform.AnyCpu, typeof(AnyCpuBenchmark)); } - private void Verify(Platform platform, Type benchmark, string failureText) + private void Verify(Platform platform, Type benchmark) { - var logger = new OutputLogger(Output); - var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry.WithPlatform(platform)) - .AddLogger(logger); // make sure we get an output in the TestRunner log - - CanExecute(benchmark, config); + .AddLogger(new OutputLogger(Output)); // make sure we get an output in the TestRunner log - var testLog = logger.GetLog(); - Assert.DoesNotContain(failureText, testLog); - Assert.DoesNotContain(BenchmarkNotFound, testLog); + // CanExecute ensures that at least one benchmark has executed successfully + CanExecute(benchmark, config, fullValidation: true); } public class X86Benchmark @@ -53,7 +42,7 @@ public void _32Bit() { if (IntPtr.Size != 4) { - throw new InvalidOperationException(X86FailedCaption); + throw new InvalidOperationException("32 bit failed!"); } } } @@ -65,7 +54,7 @@ public void _64Bit() { if (IntPtr.Size != 8) { - throw new InvalidOperationException(X64FailedCaption); + throw new InvalidOperationException("64 bit failed!"); } } } @@ -75,16 +64,6 @@ public class AnyCpuBenchmark [Benchmark] public void AnyCpu() { - Console.WriteLine(AnyCpuOkCaption); - } - } - - public class HostBenchmark - { - [Benchmark] - public void Host() - { - Console.WriteLine(HostPlatformOkCaption); } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs b/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs index 472ba3907a..74fcb3137b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs @@ -31,9 +31,8 @@ public void CanExecuteWithNonDefaultUiCulture(string culture) CultureInfo.CurrentCulture = overrideCulture; CultureInfo.CurrentUICulture = overrideCulture; - var logger = new OutputLogger(Output); var miniJob = Job.Dry.WithToolchain(RoslynToolchain.Instance); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); CanExecute(config); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/RunStrategyTests.cs b/tests/BenchmarkDotNet.IntegrationTests/RunStrategyTests.cs index 6be53ac3cf..7a5ac84a96 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RunStrategyTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RunStrategyTests.cs @@ -19,10 +19,9 @@ public RunStrategyTests(ITestOutputHelper output) : base(output) { } [Fact] public void RunStrategiesAreSupported() { - var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .AddColumnProvider(DefaultColumnProviders.Instance) - .AddLogger(logger) + .AddLogger(new OutputLogger(Output)) .AddJob(new Job(Job.Dry) { Run = { RunStrategy = RunStrategy.ColdStart } }) .AddJob(new Job(Job.Dry) { Run = { RunStrategy = RunStrategy.Monitoring } }) .AddJob(new Job(Job.Dry) { Run = { RunStrategy = RunStrategy.Throughput } }); @@ -40,45 +39,27 @@ public void RunStrategiesAreSupported() Assert.Equal(1, results.BenchmarksCases.Count(b => b.Job.Run.RunStrategy == RunStrategy.Throughput && b.Descriptor.WorkloadMethod.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.BenchmarksCases.Count(b => b.Job.Run.RunStrategy == RunStrategy.Throughput && b.Descriptor.WorkloadMethod.Name == "BenchmarkWithReturnValue")); - string testLog = logger.GetLog(); - Assert.Contains("// ### Benchmark with void called ###", testLog); - Assert.Contains("// ### Benchmark with return value called ###", testLog); - Assert.DoesNotContain("No benchmarks found", logger.GetLog()); + Assert.Equal(6, results.Reports.Length); + + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.ColdStart && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithVoid")); + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.ColdStart && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithReturnValue")); + + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.Monitoring && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithVoid")); + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.Monitoring && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithReturnValue")); + + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.Throughput && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithVoid")); + Assert.Single(results.Reports.Where(r => r.BenchmarkCase.Job.Run.RunStrategy == RunStrategy.Throughput && r.BenchmarkCase.Descriptor.WorkloadMethod.Name == "BenchmarkWithReturnValue")); + + Assert.True(results.Reports.All(r => r.AllMeasurements.Any())); } public class ModeBenchmarks { - public bool FirstTime { get; set; } - - [GlobalSetup] - public void GlobalSetup() - { - // Ensure we only print the diagnostic messages once per run in the tests, otherwise it fills up the output log!! - FirstTime = true; - } - [Benchmark] - public void BenchmarkWithVoid() - { - Thread.Sleep(10); - if (FirstTime) - { - Console.WriteLine("// ### Benchmark with void called ###"); - FirstTime = false; - } - } + public void BenchmarkWithVoid() { } [Benchmark] - public string BenchmarkWithReturnValue() - { - Thread.Sleep(10); - if (FirstTime) - { - Console.WriteLine("// ### Benchmark with return value called ###"); - FirstTime = false; - } - return "okay"; - } + public string BenchmarkWithReturnValue() => "okay"; } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/SetupAndCleanupTests.cs b/tests/BenchmarkDotNet.IntegrationTests/SetupAndCleanupTests.cs index bd6352fe32..a915c2d60b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/SetupAndCleanupTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/SetupAndCleanupTests.cs @@ -78,21 +78,21 @@ public SetupAndCleanupTests(ITestOutputHelper output) : base(output) { } [Fact] public void AllSetupAndCleanupMethodRunsForSpecificBenchmark() { - var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); - var config = CreateSimpleConfig(logger, miniJob); + var config = CreateSimpleConfig(job: miniJob); - CanExecute(config); + var summary = CanExecute(config); + var standardOutput = GetCombinedStandardOutput(summary); Output.WriteLine(OutputDelimiter); Output.WriteLine(OutputDelimiter); Output.WriteLine(OutputDelimiter); - var firstActualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(FirstPrefix)).ToArray(); + var firstActualLogLines = standardOutput.Where(line => line.StartsWith(FirstPrefix)).ToArray(); foreach (string line in firstActualLogLines) Output.WriteLine(line); Assert.Equal(firstExpectedLogLines, firstActualLogLines); - var secondActualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(SecondPrefix)).ToArray(); + var secondActualLogLines = standardOutput.Where(line => line.StartsWith(SecondPrefix)).ToArray(); foreach (string line in secondActualLogLines) Output.WriteLine(line); Assert.Equal(secondExpectedLogLines, secondActualLogLines); diff --git a/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs index 262b0facac..f8cb9686f3 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs @@ -25,7 +25,7 @@ public GenerateResult GenerateProject(BuildPartition buildPartition, ILogger log { logger.WriteLine("Generating"); Done = true; - return new GenerateResult(null, true, null, Array.Empty(), false); + return new GenerateResult(null, true, null, Array.Empty()); } } @@ -49,7 +49,7 @@ public ExecuteResult Execute(ExecuteParameters executeParameters) { executeParameters.Logger.WriteLine("Executing"); Done = true; - return new ExecuteResult(true, 0, default, Array.Empty(), Array.Empty(), executeParameters.LaunchIndex); + return new ExecuteResult(true, 0, default, Array.Empty(), Array.Empty(), Array.Empty(), executeParameters.LaunchIndex); } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs index 6fd41689f5..f6a5d10428 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ValuesReturnedByBenchmarkTest.cs @@ -122,6 +122,23 @@ public class Job { } [Benchmark] public nuint UnsignedNativeSizeInteger() => 0; + + [Benchmark] + public Tuple BenchmarkInnerClass() => Tuple.Create(new Outer(), new Outer.Inner()); + + [Benchmark] + public Tuple> BenchmarkGenericInnerClass() => Tuple.Create(new Outer(), new Outer.InnerGeneric()); + } + + public class Outer + { + public class Inner + { + } + + public class InnerGeneric + { + } } } } diff --git a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs index 227cc3ae19..978da2a516 100644 --- a/tests/BenchmarkDotNet.Tests/BuildResultTests.cs +++ b/tests/BenchmarkDotNet.Tests/BuildResultTests.cs @@ -54,7 +54,7 @@ public void UnknownMSBuildErrorsAreNotTranslatedToMoreUserFriendlyVersions() [AssertionMethod] private void Verify(string msbuildError, bool expectedResult, string expectedReason) { - var sut = BuildResult.Failure(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false), msbuildError); + var sut = BuildResult.Failure(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty()), msbuildError); Assert.Equal(expectedResult, sut.TryToExplainFailureReason(out string reason)); Assert.Equal(expectedReason, reason); diff --git a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs index 89c6fa05bd..19a0e0583f 100644 --- a/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Engine/EngineFactoryTests.cs @@ -247,7 +247,7 @@ private EngineParameters CreateEngineParameters(Action mainNoUnroll, Actio Dummy3Action = () => { }, GlobalSetupAction = GlobalSetup, GlobalCleanupAction = GlobalCleanup, - Host = new ConsoleHost(TextWriter.Null, StreamReader.Null), + Host = new NoAcknowledgementConsoleHost(), OverheadActionUnroll = OverheadUnroll, OverheadActionNoUnroll = OverheadNoUnroll, IterationCleanupAction = IterationCleanup, diff --git a/tests/BenchmarkDotNet.Tests/Loggers/OutputLogger.cs b/tests/BenchmarkDotNet.Tests/Loggers/OutputLogger.cs index 6cdd64008b..4cffd8dad4 100644 --- a/tests/BenchmarkDotNet.Tests/Loggers/OutputLogger.cs +++ b/tests/BenchmarkDotNet.Tests/Loggers/OutputLogger.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Loggers; using Xunit.Abstractions; @@ -14,12 +15,14 @@ public OutputLogger(ITestOutputHelper testOutputHelper) this.testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper)); } + [MethodImpl(MethodImplOptions.Synchronized)] public override void Write(LogKind logKind, string text) { currentLine += text; base.Write(logKind, text); } + [MethodImpl(MethodImplOptions.Synchronized)] public override void WriteLine() { testOutputHelper.WriteLine(currentLine); @@ -27,6 +30,7 @@ public override void WriteLine() base.WriteLine(); } + [MethodImpl(MethodImplOptions.Synchronized)] public override void WriteLine(LogKind logKind, string text) { testOutputHelper.WriteLine(currentLine + text); diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index aca1026861..813d25f4d1 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -67,17 +67,17 @@ private static BenchmarkCase[] CreateBenchmarks(IConfig config) private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int n, double nanoseconds) { - var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false)); + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); var measurements = Enumerable.Range(0, n) .Select(index => new Measurement(1, IterationMode.Workload, IterationStage.Result, index + 1, 1, nanoseconds + index).ToString()) .ToList(); - var executeResult = new ExecuteResult(true, 0, default, measurements, new[] { $"// Runtime=extra output line" }, 1); + var executeResult = new ExecuteResult(true, 0, default, measurements, new[] { $"// Runtime=extra output line" }, Array.Empty(), 1); return new BenchmarkReport(true, benchmarkCase, buildResult, buildResult, new List { executeResult }, Array.Empty()); } private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, bool hugeSd, Metric[] metrics) { - var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false)); + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); bool isFoo = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Foo"; bool isBar = benchmarkCase.Descriptor.WorkloadMethodDisplayInfo == "Bar"; var measurements = new List diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs index 6415157804..27f2db779c 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs @@ -81,7 +81,7 @@ private Summary CreateSummary(int[] values) private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int measurementValue) { - var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false)); + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); var measurements = new List { new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, measurementValue), diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs index 3b609eb636..c6e2f72191 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs @@ -102,7 +102,7 @@ private Summary CreateSummary(int[] values, RatioStyle ratioStyle, int noise) private static BenchmarkReport CreateReport(BenchmarkCase benchmarkCase, int measurementValue, int noise) { - var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false)); + var buildResult = BuildResult.Success(GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty())); var measurements = new List { new Measurement(1, IterationMode.Workload, IterationStage.Result, 1, 1, measurementValue), diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 369d1929c2..d0f8452e35 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -64,7 +64,7 @@ private static BenchmarkReport CreateFailureReport(BenchmarkCase benchmark) private static BenchmarkReport CreateSuccessReport(BenchmarkCase benchmark) { - GenerateResult generateResult = GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty(), false); + GenerateResult generateResult = GenerateResult.Success(ArtifactsPaths.Empty, Array.Empty()); BuildResult buildResult = BuildResult.Success(generateResult); var metrics = new[] { new Metric(new FakeMetricDescriptor(), Math.E) }; return new BenchmarkReport(true, benchmark, generateResult, buildResult, Array.Empty(), metrics);