master
1//! Default test runner for unit tests.
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
6const fatal = std.process.fatal;
7const testing = std.testing;
8const assert = std.debug.assert;
9const fuzz_abi = std.Build.abi.fuzz;
10
11pub const std_options: std.Options = .{
12 .logFn = log,
13};
14
15var log_err_count: usize = 0;
16var fba: std.heap.FixedBufferAllocator = .init(&fba_buffer);
17var fba_buffer: [8192]u8 = undefined;
18var stdin_buffer: [4096]u8 = undefined;
19var stdout_buffer: [4096]u8 = undefined;
20var runner_threaded_io: Io.Threaded = .init_single_threaded;
21
22/// Keep in sync with logic in `std.Build.addRunArtifact` which decides whether
23/// the test runner will communicate with the build runner via `std.zig.Server`.
24const need_simple = switch (builtin.zig_backend) {
25 .stage2_aarch64,
26 .stage2_powerpc,
27 .stage2_riscv64,
28 => true,
29 else => false,
30};
31
32pub fn main() void {
33 @disableInstrumentation();
34
35 if (builtin.cpu.arch.isSpirV()) {
36 // SPIR-V needs an special test-runner
37 return;
38 }
39
40 if (need_simple) {
41 return mainSimple() catch @panic("test failure\n");
42 }
43
44 const args = std.process.argsAlloc(fba.allocator()) catch
45 @panic("unable to parse command line args");
46
47 var listen = false;
48 var opt_cache_dir: ?[]const u8 = null;
49
50 for (args[1..]) |arg| {
51 if (std.mem.eql(u8, arg, "--listen=-")) {
52 listen = true;
53 } else if (std.mem.startsWith(u8, arg, "--seed=")) {
54 testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
55 @panic("unable to parse --seed command line argument");
56 } else if (std.mem.startsWith(u8, arg, "--cache-dir")) {
57 opt_cache_dir = arg["--cache-dir=".len..];
58 } else {
59 @panic("unrecognized command line argument");
60 }
61 }
62
63 if (builtin.fuzz) {
64 const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
65 fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
66 }
67
68 if (listen) {
69 return mainServer() catch @panic("internal test runner failure");
70 } else {
71 return mainTerminal();
72 }
73}
74
75fn mainServer() !void {
76 @disableInstrumentation();
77 var stdin_reader = std.fs.File.stdin().readerStreaming(runner_threaded_io.io(), &stdin_buffer);
78 var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer);
79 var server = try std.zig.Server.init(.{
80 .in = &stdin_reader.interface,
81 .out = &stdout_writer.interface,
82 .zig_version = builtin.zig_version_string,
83 });
84
85 if (builtin.fuzz) {
86 const coverage = fuzz_abi.fuzzer_coverage();
87 try server.serveCoverageIdMessage(
88 coverage.id,
89 coverage.runs,
90 coverage.unique,
91 coverage.seen,
92 );
93 }
94
95 while (true) {
96 const hdr = try server.receiveMessage();
97 switch (hdr.tag) {
98 .exit => {
99 return std.process.exit(0);
100 },
101 .query_test_metadata => {
102 testing.allocator_instance = .{};
103 defer if (testing.allocator_instance.deinit() == .leak) {
104 @panic("internal test runner memory leak");
105 };
106
107 var string_bytes: std.ArrayList(u8) = .empty;
108 defer string_bytes.deinit(testing.allocator);
109 try string_bytes.append(testing.allocator, 0); // Reserve 0 for null.
110
111 const test_fns = builtin.test_functions;
112 const names = try testing.allocator.alloc(u32, test_fns.len);
113 defer testing.allocator.free(names);
114 const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len);
115 defer testing.allocator.free(expected_panic_msgs);
116
117 for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| {
118 name.* = @intCast(string_bytes.items.len);
119 try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1);
120 string_bytes.appendSliceAssumeCapacity(test_fn.name);
121 string_bytes.appendAssumeCapacity(0);
122 expected_panic_msg.* = 0;
123 }
124
125 try server.serveTestMetadata(.{
126 .names = names,
127 .expected_panic_msgs = expected_panic_msgs,
128 .string_bytes = string_bytes.items,
129 });
130 },
131
132 .run_test => {
133 testing.allocator_instance = .{};
134 testing.io_instance = .init(testing.allocator);
135 log_err_count = 0;
136 const index = try server.receiveBody_u32();
137 const test_fn = builtin.test_functions[index];
138 is_fuzz_test = false;
139
140 // let the build server know we're starting the test now
141 try server.serveStringMessage(.test_started, &.{});
142
143 const TestResults = std.zig.Server.Message.TestResults;
144 const status: TestResults.Status = if (test_fn.func()) |v| s: {
145 v;
146 break :s .pass;
147 } else |err| switch (err) {
148 error.SkipZigTest => .skip,
149 else => s: {
150 if (@errorReturnTrace()) |trace| {
151 std.debug.dumpStackTrace(trace);
152 }
153 break :s .fail;
154 },
155 };
156 testing.io_instance.deinit();
157 const leak_count = testing.allocator_instance.detectLeaks();
158 testing.allocator_instance.deinitWithoutLeakChecks();
159 try server.serveTestResults(.{
160 .index = index,
161 .flags = .{
162 .status = status,
163 .fuzz = is_fuzz_test,
164 .log_err_count = std.math.lossyCast(
165 @FieldType(TestResults.Flags, "log_err_count"),
166 log_err_count,
167 ),
168 .leak_count = std.math.lossyCast(
169 @FieldType(TestResults.Flags, "leak_count"),
170 leak_count,
171 ),
172 },
173 });
174 },
175 .start_fuzzing => {
176 // This ensures that this code won't be analyzed and hence reference fuzzer symbols
177 // since they are not present.
178 if (!builtin.fuzz) unreachable;
179
180 const index = try server.receiveBody_u32();
181 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
182 const amount_or_instance = try server.receiveBody_u64();
183
184 const test_fn = builtin.test_functions[index];
185 const entry_addr = @intFromPtr(test_fn.func);
186
187 try server.serveU64Message(.fuzz_start_addr, fuzz_abi.fuzzer_unslide_address(entry_addr));
188 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
189 is_fuzz_test = false;
190 fuzz_test_index = index;
191 fuzz_mode = mode;
192 fuzz_amount_or_instance = amount_or_instance;
193
194 test_fn.func() catch |err| switch (err) {
195 error.SkipZigTest => return,
196 else => {
197 if (@errorReturnTrace()) |trace| {
198 std.debug.dumpStackTrace(trace);
199 }
200 std.debug.print("failed with error.{t}\n", .{err});
201 std.process.exit(1);
202 },
203 };
204 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
205 if (log_err_count != 0) @panic("error logs detected");
206 assert(mode != .forever);
207 std.process.exit(0);
208 },
209
210 else => {
211 std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)});
212 std.process.exit(1);
213 },
214 }
215 }
216}
217
218fn mainTerminal() void {
219 @disableInstrumentation();
220 if (builtin.fuzz) @panic("fuzz test requires server");
221
222 const test_fn_list = builtin.test_functions;
223 var ok_count: usize = 0;
224 var skip_count: usize = 0;
225 var fail_count: usize = 0;
226 var fuzz_count: usize = 0;
227 const root_node = if (builtin.fuzz) std.Progress.Node.none else std.Progress.start(.{
228 .root_name = "Test",
229 .estimated_total_items = test_fn_list.len,
230 });
231 const have_tty = std.fs.File.stderr().isTty();
232
233 var leaks: usize = 0;
234 for (test_fn_list, 0..) |test_fn, i| {
235 testing.allocator_instance = .{};
236 testing.io_instance = .init(testing.allocator);
237 defer {
238 testing.io_instance.deinit();
239 if (testing.allocator_instance.deinit() == .leak) leaks += 1;
240 }
241 testing.log_level = .warn;
242
243 const test_node = root_node.start(test_fn.name, 0);
244 if (!have_tty) {
245 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
246 }
247 is_fuzz_test = false;
248 if (test_fn.func()) |_| {
249 ok_count += 1;
250 test_node.end();
251 if (!have_tty) std.debug.print("OK\n", .{});
252 } else |err| switch (err) {
253 error.SkipZigTest => {
254 skip_count += 1;
255 if (have_tty) {
256 std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name });
257 } else {
258 std.debug.print("SKIP\n", .{});
259 }
260 test_node.end();
261 },
262 else => {
263 fail_count += 1;
264 if (have_tty) {
265 std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
266 i + 1, test_fn_list.len, test_fn.name, err,
267 });
268 } else {
269 std.debug.print("FAIL ({t})\n", .{err});
270 }
271 if (@errorReturnTrace()) |trace| {
272 std.debug.dumpStackTrace(trace);
273 }
274 test_node.end();
275 },
276 }
277 fuzz_count += @intFromBool(is_fuzz_test);
278 }
279 root_node.end();
280 if (ok_count == test_fn_list.len) {
281 std.debug.print("All {d} tests passed.\n", .{ok_count});
282 } else {
283 std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count });
284 }
285 if (log_err_count != 0) {
286 std.debug.print("{d} errors were logged.\n", .{log_err_count});
287 }
288 if (leaks != 0) {
289 std.debug.print("{d} tests leaked memory.\n", .{leaks});
290 }
291 if (fuzz_count != 0) {
292 std.debug.print("{d} fuzz tests found.\n", .{fuzz_count});
293 }
294 if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
295 std.process.exit(1);
296 }
297}
298
299pub fn log(
300 comptime message_level: std.log.Level,
301 comptime scope: @EnumLiteral(),
302 comptime format: []const u8,
303 args: anytype,
304) void {
305 @disableInstrumentation();
306 if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) {
307 log_err_count +|= 1;
308 }
309 if (@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) {
310 std.debug.print(
311 "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n",
312 args,
313 );
314 }
315}
316
317/// Simpler main(), exercising fewer language features, so that
318/// work-in-progress backends can handle it.
319pub fn mainSimple() anyerror!void {
320 @disableInstrumentation();
321 // is the backend capable of calling `std.fs.File.writeAll`?
322 const enable_write = switch (builtin.zig_backend) {
323 .stage2_aarch64, .stage2_riscv64 => true,
324 else => false,
325 };
326 // is the backend capable of calling `Io.Writer.print`?
327 const enable_print = switch (builtin.zig_backend) {
328 .stage2_aarch64, .stage2_riscv64 => true,
329 else => false,
330 };
331
332 var passed: u64 = 0;
333 var skipped: u64 = 0;
334 var failed: u64 = 0;
335
336 // we don't want to bring in File and Writer if the backend doesn't support it
337 const stdout = if (enable_write) std.fs.File.stdout() else {};
338
339 for (builtin.test_functions) |test_fn| {
340 if (enable_write) {
341 stdout.writeAll(test_fn.name) catch {};
342 stdout.writeAll("... ") catch {};
343 }
344 if (test_fn.func()) |_| {
345 if (enable_write) stdout.writeAll("PASS\n") catch {};
346 } else |err| {
347 if (err != error.SkipZigTest) {
348 if (enable_write) stdout.writeAll("FAIL\n") catch {};
349 failed += 1;
350 if (!enable_write) return err;
351 continue;
352 }
353 if (enable_write) stdout.writeAll("SKIP\n") catch {};
354 skipped += 1;
355 continue;
356 }
357 passed += 1;
358 }
359 if (enable_print) {
360 var stdout_writer = stdout.writer(&.{});
361 stdout_writer.interface.print("{} passed, {} skipped, {} failed\n", .{ passed, skipped, failed }) catch {};
362 }
363 if (failed != 0) std.process.exit(1);
364}
365
366var is_fuzz_test: bool = undefined;
367var fuzz_test_index: u32 = undefined;
368var fuzz_mode: fuzz_abi.LimitKind = undefined;
369var fuzz_amount_or_instance: u64 = undefined;
370
371pub fn fuzz(
372 context: anytype,
373 comptime testOne: fn (context: @TypeOf(context), []const u8) anyerror!void,
374 options: testing.FuzzInputOptions,
375) anyerror!void {
376 // Prevent this function from confusing the fuzzer by omitting its own code
377 // coverage from being considered.
378 @disableInstrumentation();
379
380 // Some compiler backends are not capable of handling fuzz testing yet but
381 // we still want CI test coverage enabled.
382 if (need_simple) return;
383
384 // Smoke test to ensure the test did not use conditional compilation to
385 // contradict itself by making it not actually be a fuzz test when the test
386 // is built in fuzz mode.
387 is_fuzz_test = true;
388
389 // Ensure no test failure occurred before starting fuzzing.
390 if (log_err_count != 0) @panic("error logs detected");
391
392 // libfuzzer is in a separate compilation unit so that its own code can be
393 // excluded from code coverage instrumentation. It needs a function pointer
394 // it can call for checking exactly one input. Inside this function we do
395 // our standard unit test checks such as memory leaks, and interaction with
396 // error logs.
397 const global = struct {
398 var ctx: @TypeOf(context) = undefined;
399
400 fn test_one(input: fuzz_abi.Slice) callconv(.c) void {
401 @disableInstrumentation();
402 testing.allocator_instance = .{};
403 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
404 log_err_count = 0;
405 testOne(ctx, input.toSlice()) catch |err| switch (err) {
406 error.SkipZigTest => return,
407 else => {
408 std.debug.lockStdErr();
409 if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace);
410 std.debug.print("failed with error.{t}\n", .{err});
411 std.process.exit(1);
412 },
413 };
414 if (log_err_count != 0) {
415 std.debug.lockStdErr();
416 std.debug.print("error logs detected\n", .{});
417 std.process.exit(1);
418 }
419 }
420 };
421 if (builtin.fuzz) {
422 const prev_allocator_state = testing.allocator_instance;
423 testing.allocator_instance = .{};
424 defer testing.allocator_instance = prev_allocator_state;
425
426 global.ctx = context;
427 fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
428
429 for (options.corpus) |elem|
430 fuzz_abi.fuzzer_new_input(.fromSlice(elem));
431
432 fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
433 return;
434 }
435
436 // When the unit test executable is not built in fuzz mode, only run the
437 // provided corpus.
438 for (options.corpus) |input| {
439 try testOne(context, input);
440 }
441
442 // In case there is no provided corpus, also use an empty
443 // string as a smoke test.
444 try testOne(context, "");
445}