Add an incremental API for running benchmarks

Allows to get results as they become available and show a progress bar type display, and allows to limit memory use by freeing results when they're received. Changed run() to return `!void` and got rid of the `Results` struct, it's not really needed. If people want the collection of `Result` values they can easily get them with `iterator()`. Split hooks and system info into their own sample programs
hendriknielaender · Mar 19, 2024 · 87b5402 · 87b5402
1 parent c0d82a0
commit 87b5402
Show file tree

Hide file tree

Showing 10 changed files with 262 additions and 102 deletions.
diff --git a/build.zig b/build.zig
@@ -60,8 +60,11 @@ fn setupExamples(b: *std.Build, target: std.zig.CrossTarget, optimize: std.built
     const example_names = [_][]const u8{
         "basic",
         "bubble_sort",
+        "hooks",
         "parameterised",
+        "progress",
         "sleep",
+        "systeminfo",
     };
 
     for (example_names) |example_name| {

diff --git a/examples/basic.zig b/examples/basic.zig
@@ -1,47 +1,19 @@
 const std = @import("std");
 const zbench = @import("zbench");
-const test_allocator = std.testing.allocator;
 
-fn helloWorld() []const u8 {
+fn myBenchmark(_: std.mem.Allocator) void {
     var result: usize = 0;
-    var i: usize = 0;
-    while (i < 1000) : (i += 1) {
-        const square = i * i;
-        result += square;
+    for (0..1_000_000) |i| {
+        std.mem.doNotOptimizeAway(i);
+        result += i * i;
     }
-
-    return "Hello, world!";
-}
-
-fn beforeAllHook() void {
-    std.debug.print("Starting benchmarking...\n", .{});
-}
-
-fn afterAllHook() void {
-    std.debug.print("Finished benchmarking.\n", .{});
-}
-
-fn myBenchmark(_: std.mem.Allocator) void {
-    _ = helloWorld();
 }
 
 test "bench test basic" {
     const stdout = std.io.getStdOut().writer();
-    var bench = zbench.Benchmark.init(test_allocator, .{});
+    var bench = zbench.Benchmark.init(std.testing.allocator, .{});
     defer bench.deinit();
 
-    try bench.add("My Benchmark", myBenchmark, .{
-        .iterations = 10,
-        .hooks = .{
-            .before_all = beforeAllHook,
-            .after_all = afterAllHook,
-        },
-    });
-
-    const sysinfo = try bench.getSystemInfo();
-    try std.fmt.format(stdout, "\n{}\n", .{sysinfo});
-
-    const results = try bench.run();
-    defer results.deinit();
-    try results.prettyPrint(stdout, true);
+    try bench.add("My Benchmark", myBenchmark, .{});
+    try bench.run(stdout);
 }
diff --git a/examples/bubble_sort.zig b/examples/bubble_sort.zig
@@ -27,7 +27,5 @@ test "bench test bubbleSort" {
 
     try bench.add("Bubble Sort Benchmark", myBenchmark, .{});
 
-    const results = try bench.run();
-    defer results.deinit();
-    try results.prettyPrint(stdout, true);
+    try bench.run(stdout);
 }
diff --git a/examples/hooks.zig b/examples/hooks.zig
@@ -0,0 +1,33 @@
+const std = @import("std");
+const zbench = @import("zbench");
+
+fn beforeAllHook() void {
+    std.debug.print("Starting benchmarking...\n", .{});
+}
+
+fn afterAllHook() void {
+    std.debug.print("Finished benchmarking.\n", .{});
+}
+
+fn myBenchmark(_: std.mem.Allocator) void {
+    var result: usize = 0;
+    for (0..1_000_000) |i| {
+        std.mem.doNotOptimizeAway(i);
+        result += i * i;
+    }
+}
+
+test "bench test hooks" {
+    const stdout = std.io.getStdOut().writer();
+    var bench = zbench.Benchmark.init(std.testing.allocator, .{});
+    defer bench.deinit();
+
+    try bench.add("My Benchmark", myBenchmark, .{
+        .iterations = 100,
+        .hooks = .{
+            .before_all = beforeAllHook,
+            .after_all = afterAllHook,
+        },
+    });
+    try bench.run(stdout);
+}
diff --git a/examples/parameterised.zig b/examples/parameterised.zig
@@ -11,7 +11,10 @@ const MyBenchmark = struct {
 
     pub fn run(self: MyBenchmark, _: std.mem.Allocator) void {
         var result: usize = 0;
-        for (0..self.loops) |i| result += i * i;
+        for (0..self.loops) |i| {
+            std.mem.doNotOptimizeAway(i);
+            result += i * i;
+        }
     }
 };
 
@@ -23,8 +26,6 @@ test "bench test parameterised" {
     try bench.addParam("My Benchmark 1", &MyBenchmark.init(100_000), .{});
     try bench.addParam("My Benchmark 2", &MyBenchmark.init(200_000), .{});
 
-    const results = try bench.run();
-    defer results.deinit();
     try stdout.writeAll("\n");
-    try results.prettyPrint(stdout, true);
+    try bench.run(stdout);
 }
diff --git a/examples/progress.zig b/examples/progress.zig
@@ -0,0 +1,52 @@
+const std = @import("std");
+const zbench = @import("zbench");
+const test_allocator = std.testing.allocator;
+
+fn myBenchmark1(_: std.mem.Allocator) void {
+    var result: usize = 0;
+    for (0..100_000) |i| {
+        std.mem.doNotOptimizeAway(i);
+        result += i * i;
+    }
+}
+
+fn myBenchmark2(_: std.mem.Allocator) void {
+    var result: usize = 0;
+    for (0..200_000) |i| {
+        std.mem.doNotOptimizeAway(i);
+        result += i * i;
+    }
+}
+
+test "bench test progress" {
+    const stdout = std.io.getStdOut().writer();
+    var bench = zbench.Benchmark.init(test_allocator, .{});
+    defer bench.deinit();
+
+    try bench.add("My Benchmark 1", myBenchmark1, .{});
+    try bench.add("My Benchmark 2", myBenchmark2, .{});
+
+    var progress = std.Progress{};
+    const progress_node = progress.start("", 0);
+    defer progress_node.end();
+
+    try stdout.writeAll("\n");
+    try bench.prettyPrintHeader(stdout);
+    var iter = try bench.iterator();
+    while (try iter.next()) |step| switch (step) {
+        .progress => |p| {
+            progress_node.setEstimatedTotalItems(p.total_runs);
+            progress_node.setCompletedItems(p.completed_runs);
+            progress_node.setName(p.current_name);
+            progress.maybeRefresh();
+        },
+        .result => |x| {
+            defer x.deinit();
+            progress_node.setName("");
+            progress_node.setEstimatedTotalItems(0);
+            progress_node.setCompletedItems(0);
+            progress.refresh();
+            try x.prettyPrint(stdout, true);
+        },
+    };
+}
diff --git a/examples/sleep.zig b/examples/sleep.zig
@@ -2,22 +2,14 @@ const std = @import("std");
 const zbench = @import("zbench");
 const test_allocator = std.testing.allocator;
 
-fn sooSleepy() void {
-    std.time.sleep(100_000_000);
-}
-
 fn sleepBenchmark(_: std.mem.Allocator) void {
-    _ = sooSleepy();
+    std.time.sleep(100_000_000);
 }
 
 test "bench test sleepy" {
     const stdout = std.io.getStdOut().writer();
     var bench = zbench.Benchmark.init(test_allocator, .{});
     defer bench.deinit();
-
     try bench.add("Sleep Benchmark", sleepBenchmark, .{});
-
-    const results = try bench.run();
-    defer results.deinit();
-    try results.prettyPrint(stdout, true);
+    try bench.run(stdout);
 }
diff --git a/examples/systeminfo.zig b/examples/systeminfo.zig
@@ -0,0 +1,22 @@
+const std = @import("std");
+const zbench = @import("zbench");
+
+fn myBenchmark(_: std.mem.Allocator) void {
+    var result: usize = 0;
+    for (0..1_000_000) |i| {
+        std.mem.doNotOptimizeAway(i);
+        result += i * i;
+    }
+}
+
+test "bench test system info" {
+    const stdout = std.io.getStdOut().writer();
+    var bench = zbench.Benchmark.init(std.testing.allocator, .{});
+    defer bench.deinit();
+
+    const sysinfo = try bench.getSystemInfo();
+    try std.fmt.format(stdout, "\n{}\n", .{sysinfo});
+
+    try bench.add("My Benchmark", myBenchmark, .{});
+    try bench.run(stdout);
+}
diff --git a/util/runner.zig b/util/runner.zig
@@ -36,6 +36,9 @@ const State = union(enum) {
     };
 
     const Running = struct {
+        /// Total number of iterations done.
+        iterations_count: usize,
+
         /// Number of timings still to be performed in the benchmark.
         iterations_remaining: usize,
 
@@ -62,6 +65,7 @@ pub fn init(
     } else .{
         .allocator = allocator,
         .state = .{ .running = .{
+            .iterations_count = iterations,
             .iterations_remaining = iterations,
             .timings_ns = try allocator.alloc(u64, iterations),
         } },
@@ -92,6 +96,7 @@ pub fn next(self: *Runner, ns: u64) Runner.Error!?Runner.Step {
                 if (N > MAX_N) N = MAX_N;
                 // Now run the benchmark with the adjusted N value
                 self.state = .{ .running = .{
+                    .iterations_count = N,
                     .iterations_remaining = N,
                     .timings_ns = try self.allocator.alloc(u64, N),
                 } };
@@ -125,6 +130,24 @@ pub fn abort(self: *Runner) void {
     };
 }
 
+pub const Status = struct {
+    total_runs: usize,
+    completed_runs: usize,
+};
+
+pub fn status(self: Runner) Status {
+    return switch (self.state) {
+        .preparing => |_| Status{
+            .total_runs = 0,
+            .completed_runs = 0,
+        },
+        .running => |st| Status{
+            .total_runs = st.iterations_count,
+            .completed_runs = st.iterations_count - st.iterations_remaining,
+        },
+    };
+}
+
 test "Runner" {
     var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9);
     {