master
1const std = @import("../std.zig");
2const Io = std.Io;
3const Build = std.Build;
4const Cache = Build.Cache;
5const Step = std.Build.Step;
6const assert = std.debug.assert;
7const fatal = std.process.fatal;
8const Allocator = std.mem.Allocator;
9const log = std.log;
10const Coverage = std.debug.Coverage;
11const abi = Build.abi.fuzz;
12const tty = std.Io.tty;
13
14const Fuzz = @This();
15const build_runner = @import("root");
16
17gpa: Allocator,
18io: Io,
19ttyconf: tty.Config,
20mode: Mode,
21
22/// Allocated into `gpa`.
23run_steps: []const *Step.Run,
24
25group: Io.Group,
26root_prog_node: std.Progress.Node,
27prog_node: std.Progress.Node,
28
29/// Protects `coverage_files`.
30coverage_mutex: Io.Mutex,
31coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
32
33queue_mutex: Io.Mutex,
34queue_cond: Io.Condition,
35msg_queue: std.ArrayList(Msg),
36
37pub const Mode = union(enum) {
38 forever: struct { ws: *Build.WebServer },
39 limit: Limited,
40
41 pub const Limited = struct {
42 amount: u64,
43 };
44};
45
46const Msg = union(enum) {
47 coverage: struct {
48 id: u64,
49 cumulative: struct {
50 runs: u64,
51 unique: u64,
52 coverage: u64,
53 },
54 run: *Step.Run,
55 },
56 entry_point: struct {
57 coverage_id: u64,
58 addr: u64,
59 },
60};
61
62const CoverageMap = struct {
63 mapped_memory: []align(std.heap.page_size_min) const u8,
64 coverage: Coverage,
65 source_locations: []Coverage.SourceLocation,
66 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.
67 entry_points: std.ArrayList(u32),
68 start_timestamp: i64,
69
70 fn deinit(cm: *CoverageMap, gpa: Allocator) void {
71 std.posix.munmap(cm.mapped_memory);
72 cm.coverage.deinit(gpa);
73 cm.* = undefined;
74 }
75};
76
77pub fn init(
78 gpa: Allocator,
79 io: Io,
80 ttyconf: tty.Config,
81 all_steps: []const *Build.Step,
82 root_prog_node: std.Progress.Node,
83 mode: Mode,
84) Allocator.Error!Fuzz {
85 const run_steps: []const *Step.Run = steps: {
86 var steps: std.ArrayList(*Step.Run) = .empty;
87 defer steps.deinit(gpa);
88 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
89 defer rebuild_node.end();
90 var rebuild_group: Io.Group = .init;
91 defer rebuild_group.cancel(io);
92
93 for (all_steps) |step| {
94 const run = step.cast(Step.Run) orelse continue;
95 if (run.producer == null) continue;
96 if (run.fuzz_tests.items.len == 0) continue;
97 try steps.append(gpa, run);
98 rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
99 }
100
101 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
102 rebuild_node.setEstimatedTotalItems(steps.items.len);
103 const run_steps = try gpa.dupe(*Step.Run, steps.items);
104 rebuild_group.wait(io);
105 break :steps run_steps;
106 };
107 errdefer gpa.free(run_steps);
108
109 for (run_steps) |run| {
110 assert(run.fuzz_tests.items.len > 0);
111 if (run.rebuilt_executable == null)
112 fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{});
113 }
114
115 return .{
116 .gpa = gpa,
117 .io = io,
118 .ttyconf = ttyconf,
119 .mode = mode,
120 .run_steps = run_steps,
121 .group = .init,
122 .root_prog_node = root_prog_node,
123 .prog_node = .none,
124 .coverage_files = .empty,
125 .coverage_mutex = .init,
126 .queue_mutex = .init,
127 .queue_cond = .{},
128 .msg_queue = .empty,
129 };
130}
131
132pub fn start(fuzz: *Fuzz) void {
133 const io = fuzz.io;
134 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
135
136 if (fuzz.mode == .forever) {
137 // For polling messages and sending updates to subscribers.
138 fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err|
139 fatal("unable to spawn coverage task: {t}", .{err});
140 }
141
142 for (fuzz.run_steps) |run| {
143 for (run.fuzz_tests.items) |unit_test_index| {
144 assert(run.rebuilt_executable != null);
145 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
146 }
147 }
148}
149
150pub fn deinit(fuzz: *Fuzz) void {
151 const io = fuzz.io;
152 fuzz.group.cancel(io);
153 fuzz.prog_node.end();
154 fuzz.gpa.free(fuzz.run_steps);
155}
156
157fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: tty.Config, parent_prog_node: std.Progress.Node) void {
158 rebuildTestsWorkerRunFallible(run, gpa, ttyconf, parent_prog_node) catch |err| {
159 const compile = run.producer.?;
160 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
161 };
162}
163
164fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, ttyconf: tty.Config, parent_prog_node: std.Progress.Node) !void {
165 const compile = run.producer.?;
166 const prog_node = parent_prog_node.start(compile.step.name, 0);
167 defer prog_node.end();
168
169 const result = compile.rebuildInFuzzMode(gpa, prog_node);
170
171 const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0;
172 const show_error_msgs = compile.step.result_error_msgs.items.len > 0;
173 const show_stderr = compile.step.result_stderr.len > 0;
174
175 if (show_error_msgs or show_compile_errors or show_stderr) {
176 var buf: [256]u8 = undefined;
177 const w, _ = std.debug.lockStderrWriter(&buf);
178 defer std.debug.unlockStderrWriter();
179 build_runner.printErrorMessages(gpa, &compile.step, .{}, w, ttyconf, .verbose, .indent) catch {};
180 }
181
182 const rebuilt_bin_path = result catch |err| switch (err) {
183 error.MakeFailed => return,
184 else => |other| return other,
185 };
186 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
187}
188
189fn fuzzWorkerRun(
190 fuzz: *Fuzz,
191 run: *Step.Run,
192 unit_test_index: u32,
193) void {
194 const gpa = run.step.owner.allocator;
195 const test_name = run.cached_test_metadata.?.testName(unit_test_index);
196
197 const prog_node = fuzz.prog_node.start(test_name, 0);
198 defer prog_node.end();
199
200 run.rerunInFuzzMode(fuzz, unit_test_index, prog_node) catch |err| switch (err) {
201 error.MakeFailed => {
202 var buf: [256]u8 = undefined;
203 const w, _ = std.debug.lockStderrWriter(&buf);
204 defer std.debug.unlockStderrWriter();
205 build_runner.printErrorMessages(gpa, &run.step, .{}, w, fuzz.ttyconf, .verbose, .indent) catch {};
206 return;
207 },
208 else => {
209 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, test_name, err });
210 return;
211 },
212 };
213}
214
215pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
216 assert(fuzz.mode == .forever);
217
218 var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
219 defer arena_state.deinit();
220 const arena = arena_state.allocator();
221
222 const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
223 var dedup_table: DedupTable = .empty;
224 defer dedup_table.deinit(fuzz.gpa);
225
226 for (fuzz.run_steps) |run_step| {
227 const compile_inputs = run_step.producer.?.step.inputs.table;
228 for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
229 try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
230 for (file_list.items) |sub_path| {
231 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
232 const joined_path = try dir_path.join(arena, sub_path);
233 dedup_table.putAssumeCapacity(joined_path, {});
234 }
235 }
236 }
237
238 const deduped_paths = dedup_table.keys();
239 const SortContext = struct {
240 pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool {
241 _ = this;
242 return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) {
243 .lt => true,
244 .gt => false,
245 .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path),
246 };
247 }
248 };
249 std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
250 return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
251}
252
253pub const Previous = struct {
254 unique_runs: usize,
255 entry_points: usize,
256 sent_source_index: bool,
257 pub const init: Previous = .{
258 .unique_runs = 0,
259 .entry_points = 0,
260 .sent_source_index = false,
261 };
262};
263pub fn sendUpdate(
264 fuzz: *Fuzz,
265 socket: *std.http.Server.WebSocket,
266 prev: *Previous,
267) !void {
268 const io = fuzz.io;
269
270 try fuzz.coverage_mutex.lock(io);
271 defer fuzz.coverage_mutex.unlock(io);
272
273 const coverage_maps = fuzz.coverage_files.values();
274 if (coverage_maps.len == 0) return;
275 // TODO: handle multiple fuzz steps in the WebSocket packets
276 const coverage_map = &coverage_maps[0];
277 const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
278 // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the
279 // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the
280 // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass
281 // this data straight to the socket with sendfile...
282 const seen_pcs = cov_header.seenBits();
283 const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
284 const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
285 {
286 if (!prev.sent_source_index) {
287 prev.sent_source_index = true;
288 // We need to send initial context.
289 const header: abi.SourceIndexHeader = .{
290 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
291 .files_len = @intCast(coverage_map.coverage.files.entries.len),
292 .source_locations_len = @intCast(coverage_map.source_locations.len),
293 .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len),
294 .start_timestamp = coverage_map.start_timestamp,
295 };
296 var iovecs: [5][]const u8 = .{
297 @ptrCast(&header),
298 @ptrCast(coverage_map.coverage.directories.keys()),
299 @ptrCast(coverage_map.coverage.files.keys()),
300 @ptrCast(coverage_map.source_locations),
301 coverage_map.coverage.string_bytes.items,
302 };
303 try socket.writeMessageVec(&iovecs, .binary);
304 }
305
306 const header: abi.CoverageUpdateHeader = .{
307 .n_runs = n_runs,
308 .unique_runs = unique_runs,
309 };
310 var iovecs: [2][]const u8 = .{
311 @ptrCast(&header),
312 @ptrCast(seen_pcs),
313 };
314 try socket.writeMessageVec(&iovecs, .binary);
315
316 prev.unique_runs = unique_runs;
317 }
318
319 if (prev.entry_points != coverage_map.entry_points.items.len) {
320 const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len));
321 var iovecs: [2][]const u8 = .{
322 @ptrCast(&header),
323 @ptrCast(coverage_map.entry_points.items),
324 };
325 try socket.writeMessageVec(&iovecs, .binary);
326
327 prev.entry_points = coverage_map.entry_points.items.len;
328 }
329}
330
331fn coverageRun(fuzz: *Fuzz) void {
332 coverageRunCancelable(fuzz) catch |err| switch (err) {
333 error.Canceled => return,
334 };
335}
336
337fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
338 const io = fuzz.io;
339
340 try fuzz.queue_mutex.lock(io);
341 defer fuzz.queue_mutex.unlock(io);
342
343 while (true) {
344 try fuzz.queue_cond.wait(io, &fuzz.queue_mutex);
345 for (fuzz.msg_queue.items) |msg| switch (msg) {
346 .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) {
347 error.AlreadyReported => continue,
348 error.Canceled => return,
349 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
350 },
351 .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) {
352 error.AlreadyReported => continue,
353 error.Canceled => return,
354 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
355 },
356 };
357 fuzz.msg_queue.clearRetainingCapacity();
358 }
359}
360fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
361 assert(fuzz.mode == .forever);
362 const ws = fuzz.mode.forever.ws;
363 const io = fuzz.io;
364
365 try fuzz.coverage_mutex.lock(io);
366 defer fuzz.coverage_mutex.unlock(io);
367
368 const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
369 if (gop.found_existing) {
370 // We are fuzzing the same executable with multiple threads.
371 // Perhaps the same unit test; perhaps a different one. In any
372 // case, since the coverage file is the same, we only have to
373 // notice changes to that one file in order to learn coverage for
374 // this particular executable.
375 return;
376 }
377 errdefer _ = fuzz.coverage_files.pop();
378
379 gop.value_ptr.* = .{
380 .coverage = std.debug.Coverage.init,
381 .mapped_memory = undefined, // populated below
382 .source_locations = undefined, // populated below
383 .entry_points = .{},
384 .start_timestamp = ws.now(),
385 };
386 errdefer gop.value_ptr.coverage.deinit(fuzz.gpa);
387
388 const rebuilt_exe_path = run_step.rebuilt_executable.?;
389 const target = run_step.producer.?.rootModuleTarget();
390 var debug_info = std.debug.Info.load(
391 fuzz.gpa,
392 rebuilt_exe_path,
393 &gop.value_ptr.coverage,
394 target.ofmt,
395 target.cpu.arch,
396 ) catch |err| {
397 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
398 run_step.step.name, rebuilt_exe_path, err,
399 });
400 return error.AlreadyReported;
401 };
402 defer debug_info.deinit(fuzz.gpa);
403
404 const coverage_file_path: Build.Cache.Path = .{
405 .root_dir = run_step.step.owner.cache_root,
406 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
407 };
408 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
409 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
410 run_step.step.name, coverage_file_path, err,
411 });
412 return error.AlreadyReported;
413 };
414 defer coverage_file.close();
415
416 const file_size = coverage_file.getEndPos() catch |err| {
417 log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err });
418 return error.AlreadyReported;
419 };
420
421 const mapped_memory = std.posix.mmap(
422 null,
423 file_size,
424 std.posix.PROT.READ,
425 .{ .TYPE = .SHARED },
426 coverage_file.handle,
427 0,
428 ) catch |err| {
429 log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err });
430 return error.AlreadyReported;
431 };
432 gop.value_ptr.mapped_memory = mapped_memory;
433
434 const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
435 const pcs = header.pcAddrs();
436 const source_locations = try fuzz.gpa.alloc(Coverage.SourceLocation, pcs.len);
437 errdefer fuzz.gpa.free(source_locations);
438
439 // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
440 // counters feature is not sorted.
441 var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};
442 defer sorted_pcs.deinit(fuzz.gpa);
443 try sorted_pcs.resize(fuzz.gpa, pcs.len);
444 @memcpy(sorted_pcs.items(.pc), pcs);
445 for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
446 sorted_pcs.sortUnstable(struct {
447 addrs: []const u64,
448
449 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
450 return ctx.addrs[a_index] < ctx.addrs[b_index];
451 }
452 }{ .addrs = sorted_pcs.items(.pc) });
453
454 debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
455 log.err("failed to resolve addresses to source locations: {t}", .{err});
456 return error.AlreadyReported;
457 };
458
459 for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl;
460 gop.value_ptr.source_locations = source_locations;
461
462 ws.notifyUpdate();
463}
464
465fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void {
466 const io = fuzz.io;
467
468 try fuzz.coverage_mutex.lock(io);
469 defer fuzz.coverage_mutex.unlock(io);
470
471 const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?;
472 const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
473 const pcs = header.pcAddrs();
474
475 // Since this pcs list is unsorted, we must linear scan for the best index.
476 const index = i: {
477 var best: usize = 0;
478 for (pcs[1..], 1..) |elem_addr, i| {
479 if (elem_addr == addr) break :i i;
480 if (elem_addr > addr) continue;
481 if (elem_addr > pcs[best]) best = i;
482 }
483 break :i best;
484 };
485 if (index >= pcs.len) {
486 log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{
487 addr, pcs[0], pcs[pcs.len - 1],
488 });
489 return error.AlreadyReported;
490 }
491 if (false) {
492 const sl = coverage_map.source_locations[index];
493 const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename);
494 if (pcs.len == 1) {
495 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{
496 addr, file_name, sl.line, sl.column,
497 });
498 } else if (index == 0) {
499 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{
500 addr, file_name, sl.line, sl.column, pcs[index + 1],
501 });
502 } else if (index == pcs.len - 1) {
503 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{
504 addr, file_name, sl.line, sl.column, index, pcs[index - 1],
505 });
506 } else {
507 log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{
508 addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
509 });
510 }
511 }
512 try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
513}
514
515pub fn waitAndPrintReport(fuzz: *Fuzz) void {
516 assert(fuzz.mode == .limit);
517 const io = fuzz.io;
518
519 fuzz.group.wait(io);
520 fuzz.group = .init;
521
522 std.debug.print("======= FUZZING REPORT =======\n", .{});
523 for (fuzz.msg_queue.items) |msg| {
524 if (msg != .coverage) continue;
525
526 const cov = msg.coverage;
527 const coverage_file_path: std.Build.Cache.Path = .{
528 .root_dir = cov.run.step.owner.cache_root,
529 .sub_path = "v/" ++ std.fmt.hex(cov.id),
530 };
531 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
532 fatal("step '{s}': failed to load coverage file '{f}': {t}", .{
533 cov.run.step.name, coverage_file_path, err,
534 });
535 };
536 defer coverage_file.close();
537
538 const fuzz_abi = std.Build.abi.fuzz;
539 var rbuf: [0x1000]u8 = undefined;
540 var r = coverage_file.reader(io, &rbuf);
541
542 var header: fuzz_abi.SeenPcsHeader = undefined;
543 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
544 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
545 cov.run.step.name, coverage_file_path, err,
546 });
547 };
548
549 if (header.pcs_len == 0) {
550 fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
551 cov.run.step.name, coverage_file_path,
552 });
553 }
554
555 var seen_count: usize = 0;
556 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
557 for (0..chunk_count) |_| {
558 const seen = r.interface.takeInt(usize, .little) catch |err| {
559 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
560 cov.run.step.name, coverage_file_path, err,
561 });
562 };
563 seen_count += @popCount(seen);
564 }
565
566 const seen_f: f64 = @floatFromInt(seen_count);
567 const total_f: f64 = @floatFromInt(header.pcs_len);
568 const ratio = seen_f / total_f;
569 std.debug.print(
570 \\Step: {s}
571 \\Fuzz test: "{s}" ({x})
572 \\Runs: {} -> {}
573 \\Unique runs: {} -> {}
574 \\Coverage: {}/{} -> {}/{} ({:.02}%)
575 \\
576 , .{
577 cov.run.step.name,
578 cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
579 cov.id,
580 cov.cumulative.runs,
581 header.n_runs,
582 cov.cumulative.unique,
583 header.unique_runs,
584 cov.cumulative.coverage,
585 header.pcs_len,
586 seen_count,
587 header.pcs_len,
588 ratio * 100,
589 });
590
591 std.debug.print("------------------------------\n", .{});
592 }
593 std.debug.print(
594 \\Values are accumulated across multiple runs when preserving the cache.
595 \\==============================
596 \\
597 , .{});
598}