Commit bce3b1efb0

@@ -401,7 +401,7 @@ pub fn main() !void {
             else => return err,
         };
         if (fuzz) {
-            Fuzz.start(&run.thread_pool, run.step_stack.keys(), main_progress_node);
+            Fuzz.start(&run.thread_pool, run.step_stack.keys(), run.ttyconf, main_progress_node);
         }
 
         if (!watch) return cleanExit();
@@ -1072,7 +1072,7 @@ fn workerMakeOneStep(
         std.debug.lockStdErr();
         defer std.debug.unlockStdErr();
 
-        printErrorMessages(b, s, run) catch {};
+        printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
     }
 
     handle_result: {
@@ -1125,10 +1125,14 @@ fn workerMakeOneStep(
     }
 }
 
-fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void {
+pub fn printErrorMessages(
+    b: *std.Build,
+    failing_step: *Step,
+    ttyconf: std.io.tty.Config,
+    stderr: File,
+    prominent_compile_errors: bool,
+) !void {
     const gpa = b.allocator;
-    const stderr = run.stderr;
-    const ttyconf = run.ttyconf;
 
     // Provide context for where these error messages are coming from by
     // printing the corresponding Step subtree.
@@ -1166,7 +1170,7 @@ fn printErrorMessages(b: *std.Build, failing_step: *Step, run: *const Run) !void
         }
     }
 
-    if (!run.prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
+    if (!prominent_compile_errors and failing_step.result_error_bundle.errorMessageCount() > 0)
         try failing_step.result_error_bundle.renderToWriter(renderOptions(ttyconf), stderr.writer());
 
     for (failing_step.result_error_msgs.items) |msg| {

@@ -89,6 +89,8 @@ has_side_effects: bool,
 /// If this is a Zig unit test binary, this tracks the indexes of the unit
 /// tests that are also fuzz tests.
 fuzz_tests: std.ArrayListUnmanaged(u32),
+cached_test_metadata: ?CachedTestMetadata = null,
+
 /// Populated during the fuzz phase if this run step corresponds to a unit test
 /// executable that contains fuzz tests.
 rebuilt_executable: ?[]const u8,
@@ -754,7 +756,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
                 b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
         }
 
-        try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node);
+        try runCommand(run, argv_list.items, has_side_effects, output_dir_path, prog_node, null);
         if (!has_side_effects) try step.writeManifestAndWatch(&man);
         return;
     };
@@ -784,7 +786,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
             b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
     }
 
-    try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node);
+    try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, null);
 
     const dep_file_dir = std.fs.cwd();
     const dep_file_basename = dep_output_file.generated_file.getPath();
@@ -843,6 +845,38 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     );
 }
 
+pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.Node) !void {
+    const step = &run.step;
+    const b = step.owner;
+    const arena = b.allocator;
+    var argv_list: std.ArrayListUnmanaged([]const u8) = .{};
+    for (run.argv.items) |arg| {
+        switch (arg) {
+            .bytes => |bytes| {
+                try argv_list.append(arena, bytes);
+            },
+            .lazy_path => |file| {
+                const file_path = file.lazy_path.getPath2(b, step);
+                try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
+            },
+            .directory_source => |file| {
+                const file_path = file.lazy_path.getPath2(b, step);
+                try argv_list.append(arena, b.fmt("{s}{s}", .{ file.prefix, file_path }));
+            },
+            .artifact => |pa| {
+                const artifact = pa.artifact;
+                const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
+                try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
+            },
+            .output_file, .output_directory => unreachable,
+        }
+    }
+    const has_side_effects = false;
+    const rand_int = std.crypto.random.int(u64);
+    const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
+    try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, prog_node, unit_test_index);
+}
+
 fn populateGeneratedPaths(
     arena: std.mem.Allocator,
     output_placeholders: []const IndexedOutput,
@@ -921,6 +955,7 @@ fn runCommand(
     has_side_effects: bool,
     output_dir_path: []const u8,
     prog_node: std.Progress.Node,
+    fuzz_unit_test_index: ?u32,
 ) !void {
     const step = &run.step;
     const b = step.owner;
@@ -939,7 +974,7 @@ fn runCommand(
     var interp_argv = std.ArrayList([]const u8).init(b.allocator);
     defer interp_argv.deinit();
 
-    const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node) catch |err| term: {
+    const result = spawnChildAndCollect(run, argv, has_side_effects, prog_node, fuzz_unit_test_index) catch |err| term: {
         // InvalidExe: cpu arch mismatch
         // FileNotFound: can happen with a wrong dynamic linker path
         if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
@@ -1075,7 +1110,7 @@ fn runCommand(
 
             try Step.handleVerbose2(step.owner, cwd, run.env_map, interp_argv.items);
 
-            break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node) catch |e| {
+            break :term spawnChildAndCollect(run, interp_argv.items, has_side_effects, prog_node, fuzz_unit_test_index) catch |e| {
                 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
 
                 return step.fail("unable to spawn interpreter {s}: {s}", .{
@@ -1090,6 +1125,15 @@ fn runCommand(
     step.result_duration_ns = result.elapsed_ns;
     step.result_peak_rss = result.peak_rss;
     step.test_results = result.stdio.test_results;
+    if (result.stdio.test_metadata) |tm|
+        run.cached_test_metadata = tm.toCachedTestMetadata();
+
+    const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
+
+    if (fuzz_unit_test_index != null) {
+        try step.handleChildProcessTerm(result.term, cwd, final_argv);
+        return;
+    }
 
     // Capture stdout and stderr to GeneratedFile objects.
     const Stream = struct {
@@ -1126,8 +1170,6 @@ fn runCommand(
         }
     }
 
-    const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
-
     switch (run.stdio) {
         .check => |checks| for (checks.items) |check| switch (check) {
             .expect_stderr_exact => |expected_bytes| {
@@ -1253,10 +1295,16 @@ fn spawnChildAndCollect(
     argv: []const []const u8,
     has_side_effects: bool,
     prog_node: std.Progress.Node,
+    fuzz_unit_test_index: ?u32,
 ) !ChildProcResult {
     const b = run.step.owner;
     const arena = b.allocator;
 
+    if (fuzz_unit_test_index != null) {
+        assert(!has_side_effects);
+        assert(run.stdio == .zig_test);
+    }
+
     var child = std.process.Child.init(argv, arena);
     if (run.cwd) |lazy_cwd| {
         child.cwd = lazy_cwd.getPath2(b, &run.step);
@@ -1306,7 +1354,7 @@ fn spawnChildAndCollect(
         var timer = try std.time.Timer.start();
 
         const result = if (run.stdio == .zig_test)
-            evalZigTest(run, &child, prog_node)
+            evalZigTest(run, &child, prog_node, fuzz_unit_test_index)
         else
             evalGeneric(run, &child);
 
@@ -1332,6 +1380,7 @@ fn evalZigTest(
     run: *Run,
     child: *std.process.Child,
     prog_node: std.Progress.Node,
+    fuzz_unit_test_index: ?u32,
 ) !StdIoResult {
     const gpa = run.step.owner.allocator;
     const arena = run.step.owner.allocator;
@@ -1342,7 +1391,12 @@ fn evalZigTest(
     });
     defer poller.deinit();
 
-    try sendMessage(child.stdin.?, .query_test_metadata);
+    if (fuzz_unit_test_index) |index| {
+        try sendRunTestMessage(child.stdin.?, .start_fuzzing, index);
+    } else {
+        run.fuzz_tests.clearRetainingCapacity();
+        try sendMessage(child.stdin.?, .query_test_metadata);
+    }
 
     const Header = std.zig.Server.Message.Header;
 
@@ -1360,8 +1414,6 @@ fn evalZigTest(
     var sub_prog_node: ?std.Progress.Node = null;
     defer if (sub_prog_node) |n| n.end();
 
-    run.fuzz_tests.clearRetainingCapacity();
-
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
             if (!(try poller.poll())) break :poll;
@@ -1382,6 +1434,7 @@ fn evalZigTest(
                 }
             },
             .test_metadata => {
+                assert(fuzz_unit_test_index == null);
                 const TmHdr = std.zig.Server.Message.TestMetadata;
                 const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));
                 test_count = tm_hdr.tests_len;
@@ -1410,6 +1463,7 @@ fn evalZigTest(
                 try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node);
             },
             .test_results => {
+                assert(fuzz_unit_test_index == null);
                 const md = metadata.?;
 
                 const TrHdr = std.zig.Server.Message.TestResults;
@@ -1479,7 +1533,23 @@ const TestMetadata = struct {
     next_index: u32,
     prog_node: std.Progress.Node,
 
+    fn toCachedTestMetadata(tm: TestMetadata) CachedTestMetadata {
+        return .{
+            .names = tm.names,
+            .string_bytes = tm.string_bytes,
+        };
+    }
+
     fn testName(tm: TestMetadata, index: u32) []const u8 {
+        return tm.toCachedTestMetadata().testName(index);
+    }
+};
+
+pub const CachedTestMetadata = struct {
+    names: []const u32,
+    string_bytes: []const u8,
+
+    pub fn testName(tm: CachedTestMetadata, index: u32) []const u8 {
         return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0);
     }
 };
@@ -1495,7 +1565,7 @@ fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Pr
         if (sub_prog_node.*) |n| n.end();
         sub_prog_node.* = metadata.prog_node.start(name, 0);
 
-        try sendRunTestMessage(in, i);
+        try sendRunTestMessage(in, .run_test, i);
         return;
     } else {
         try sendMessage(in, .exit);
@@ -1510,9 +1580,9 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
     try file.writeAll(std.mem.asBytes(&header));
 }
 
-fn sendRunTestMessage(file: std.fs.File, index: u32) !void {
+fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void {
     const header: std.zig.Client.Message.Header = .{
-        .tag = .run_test,
+        .tag = tag,
         .bytes_len = 4,
     };
     const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index);

@@ -3,9 +3,15 @@ const Fuzz = @This();
 const Step = std.Build.Step;
 const assert = std.debug.assert;
 const fatal = std.process.fatal;
+const build_runner = @import("root");
 
-pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void {
-    {
+pub fn start(
+    thread_pool: *std.Thread.Pool,
+    all_steps: []const *Step,
+    ttyconf: std.io.tty.Config,
+    prog_node: std.Progress.Node,
+) void {
+    const count = block: {
         const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0);
         defer rebuild_node.end();
         var count: usize = 0;
@@ -14,13 +20,14 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
         for (all_steps) |step| {
             const run = step.cast(Step.Run) orelse continue;
             if (run.fuzz_tests.items.len > 0 and run.producer != null) {
-                thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node });
+                thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, ttyconf, rebuild_node });
                 count += 1;
             }
         }
         if (count == 0) fatal("no fuzz tests found", .{});
         rebuild_node.setEstimatedTotalItems(count);
-    }
+        break :block count;
+    };
 
     // Detect failure.
     for (all_steps) |step| {
@@ -29,18 +36,69 @@ pub fn start(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node:
             fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
     }
 
-    @panic("TODO do something with the rebuilt unit tests");
+    {
+        const rebuild_node = prog_node.start("Fuzzing", count);
+        defer rebuild_node.end();
+        var wait_group: std.Thread.WaitGroup = .{};
+        defer wait_group.wait();
+
+        for (all_steps) |step| {
+            const run = step.cast(Step.Run) orelse continue;
+            for (run.fuzz_tests.items) |unit_test_index| {
+                assert(run.rebuilt_executable != null);
+                thread_pool.spawnWg(&wait_group, fuzzWorkerRun, .{ run, unit_test_index, ttyconf, prog_node });
+            }
+        }
+    }
+
+    fatal("all fuzz workers crashed", .{});
 }
 
-fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void {
+fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
     const compile_step = run.producer.?;
     const prog_node = parent_prog_node.start(compile_step.step.name, 0);
     defer prog_node.end();
-    const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| {
-        std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{
-            compile_step.step.name, @errorName(err),
-        });
-        return;
+    if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| {
+        run.rebuilt_executable = rebuilt_bin_path;
+    } else |err| switch (err) {
+        error.MakeFailed => {
+            const b = run.step.owner;
+            const stderr = std.io.getStdErr();
+            std.debug.lockStdErr();
+            defer std.debug.unlockStdErr();
+            build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {};
+        },
+        else => {
+            std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{
+                compile_step.step.name, @errorName(err),
+            });
+        },
+    }
+}
+
+fn fuzzWorkerRun(
+    run: *Step.Run,
+    unit_test_index: u32,
+    ttyconf: std.io.tty.Config,
+    parent_prog_node: std.Progress.Node,
+) void {
+    const test_name = run.cached_test_metadata.?.testName(unit_test_index);
+
+    const prog_node = parent_prog_node.start(test_name, 0);
+    defer prog_node.end();
+
+    run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) {
+        error.MakeFailed => {
+            const b = run.step.owner;
+            const stderr = std.io.getStdErr();
+            std.debug.lockStdErr();
+            defer std.debug.unlockStdErr();
+            build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {};
+        },
+        else => {
+            std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{
+                run.step.name, test_name, @errorName(err),
+            });
+        },
     };
-    run.rebuilt_executable = rebuilt_bin_path;
 }

@@ -33,6 +33,9 @@ pub const Message = struct {
         /// Ask the test runner to run a particular test.
         /// The message body is a u32 test index.
         run_test,
+        /// Ask the test runner to start fuzzing a particular test.
+        /// The message body is a u32 test index.
+        start_fuzzing,
 
         _,
     };