master
1const Cases = @This();
2const builtin = @import("builtin");
3const std = @import("std");
4const assert = std.debug.assert;
5const Allocator = std.mem.Allocator;
6const getExternalExecutor = std.zig.system.getExternalExecutor;
7const ArrayList = std.ArrayList;
8
9gpa: Allocator,
10arena: Allocator,
11cases: std.array_list.Managed(Case),
12
13pub const IncrementalCase = struct {
14 base_path: []const u8,
15};
16
17pub const File = struct {
18 src: [:0]const u8,
19 path: []const u8,
20};
21
22pub const DepModule = struct {
23 name: []const u8,
24 path: []const u8,
25};
26
27pub const Backend = enum {
28 /// Test does not care which backend is used; compiler gets to pick the default.
29 auto,
30 selfhosted,
31 llvm,
32};
33
34pub const CFrontend = enum {
35 clang,
36 aro,
37};
38
39pub const Case = struct {
40 /// The name of the test case. This is shown if a test fails, and
41 /// otherwise ignored.
42 name: []const u8,
43 /// The platform the test targets. For non-native platforms, an emulator
44 /// such as QEMU is required for tests to complete.
45 target: std.Build.ResolvedTarget,
46 /// In order to be able to run e.g. Execution updates, this must be set
47 /// to Executable.
48 output_mode: std.builtin.OutputMode,
49 optimize_mode: std.builtin.OptimizeMode = .Debug,
50
51 files: std.array_list.Managed(File),
52 case: ?union(enum) {
53 /// Check that it compiles with no errors.
54 Compile: void,
55 /// Check the main binary output file against an expected set of bytes.
56 /// This is most useful with, for example, `-ofmt=c`.
57 CompareObjectFile: []const u8,
58 /// An error update attempts to compile bad code, and ensures that it
59 /// fails to compile, and for the expected reasons.
60 /// A slice containing the expected stderr template, which
61 /// gets some values substituted.
62 Error: []const []const u8,
63 /// An execution update compiles and runs the input, testing the
64 /// stdout against the expected results
65 /// This is a slice containing the expected message.
66 Execution: []const u8,
67 /// A header update compiles the input with the equivalent of
68 /// `-femit-h` and tests the produced header against the
69 /// expected result.
70 Header: []const u8,
71 },
72
73 emit_asm: bool = false,
74 emit_bin: bool = true,
75 emit_h: bool = false,
76 is_test: bool = false,
77 expect_exact: bool = false,
78 backend: Backend = .auto,
79 link_libc: bool = false,
80 pic: ?bool = null,
81 pie: ?bool = null,
82 /// A list of imports to cache alongside the source file.
83 imports: []const []const u8 = &.{},
84 /// Where to look for imports relative to the `cases_dir_path` given to
85 /// `lower_to_build_steps`. If null, file imports will assert.
86 import_path: ?[]const u8 = null,
87
88 deps: std.array_list.Managed(DepModule),
89
90 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
91 case.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
92 }
93
94 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void {
95 case.deps.append(.{
96 .name = name,
97 .path = path,
98 }) catch @panic("out of memory");
99 }
100
101 /// Adds a subcase in which the module is updated with `src`, compiled,
102 /// run, and the output is tested against `result`.
103 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {
104 assert(self.case == null);
105 self.case = .{ .Execution = result };
106 self.addSourceFile("tmp.zig", src);
107 }
108
109 /// Adds a subcase in which the module is updated with `src`, which
110 /// should contain invalid input, and ensures that compilation fails
111 /// for the expected reasons, given in sequential order in `errors` in
112 /// the form `:line:column: error: message`.
113 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
114 assert(errors.len != 0);
115 assert(self.case == null);
116 self.case = .{ .Error = errors };
117 self.addSourceFile("tmp.zig", src);
118 }
119
120 /// Adds a subcase in which the module is updated with `src`, and
121 /// asserts that it compiles without issue
122 pub fn addCompile(self: *Case, src: [:0]const u8) void {
123 assert(self.case == null);
124 self.case = .Compile;
125 self.addSourceFile("tmp.zig", src);
126 }
127};
128
129pub fn addExe(
130 ctx: *Cases,
131 name: []const u8,
132 target: std.Build.ResolvedTarget,
133) *Case {
134 ctx.cases.append(.{
135 .name = name,
136 .target = target,
137 .files = .init(ctx.arena),
138 .case = null,
139 .output_mode = .Exe,
140 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
141 }) catch @panic("out of memory");
142 return &ctx.cases.items[ctx.cases.items.len - 1];
143}
144
145/// Adds a test case for Zig input, producing an executable
146pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
147 return ctx.addExe(name, target);
148}
149
150pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case {
151 var adjusted_query = target_query;
152 adjusted_query.ofmt = .c;
153 ctx.cases.append(.{
154 .name = name,
155 .target = b.resolveTargetQuery(adjusted_query),
156 .files = .init(ctx.arena),
157 .case = null,
158 .output_mode = .Exe,
159 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
160 .link_libc = true,
161 }) catch @panic("out of memory");
162 return &ctx.cases.items[ctx.cases.items.len - 1];
163}
164
165pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
166 const can_emit_asm = switch (target.result.cpu.arch) {
167 .csky,
168 .xtensa,
169 => false,
170 else => true,
171 };
172 const can_emit_bin = switch (target.result.cpu.arch) {
173 .arc,
174 .csky,
175 .nvptx,
176 .nvptx64,
177 .xcore,
178 .xtensa,
179 => false,
180 else => true,
181 };
182
183 ctx.cases.append(.{
184 .name = name,
185 .target = target,
186 .files = .init(ctx.arena),
187 .case = null,
188 .output_mode = .Obj,
189 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
190 .backend = .llvm,
191 .emit_bin = can_emit_bin,
192 .emit_asm = can_emit_asm,
193 }) catch @panic("out of memory");
194 return &ctx.cases.items[ctx.cases.items.len - 1];
195}
196
197pub fn addObj(
198 ctx: *Cases,
199 name: []const u8,
200 target: std.Build.ResolvedTarget,
201) *Case {
202 ctx.cases.append(.{
203 .name = name,
204 .target = target,
205 .files = .init(ctx.arena),
206 .case = null,
207 .output_mode = .Obj,
208 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
209 }) catch @panic("out of memory");
210 return &ctx.cases.items[ctx.cases.items.len - 1];
211}
212
213pub fn addTest(
214 ctx: *Cases,
215 name: []const u8,
216 target: std.Build.ResolvedTarget,
217) *Case {
218 ctx.cases.append(.{
219 .name = name,
220 .target = target,
221 .files = .init(ctx.arena),
222 .case = null,
223 .output_mode = .Exe,
224 .is_test = true,
225 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
226 }) catch @panic("out of memory");
227 return &ctx.cases.items[ctx.cases.items.len - 1];
228}
229
230/// Adds a test case for Zig input, producing an object file.
231pub fn obj(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
232 return ctx.addObj(name, target);
233}
234
235/// Adds a test case for ZIR input, producing an object file.
236pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
237 return ctx.addObj(name, target, .ZIR);
238}
239
240/// Adds a test case for Zig or ZIR input, producing C code.
241pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
242 var target_adjusted = target;
243 target_adjusted.ofmt = std.Target.ObjectFormat.c;
244 ctx.cases.append(.{
245 .name = name,
246 .target = target_adjusted,
247 .files = .init(ctx.arena),
248 .case = null,
249 .output_mode = .Obj,
250 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
251 }) catch @panic("out of memory");
252 return &ctx.cases.items[ctx.cases.items.len - 1];
253}
254
255pub fn addTransform(
256 ctx: *Cases,
257 name: []const u8,
258 target: std.Build.ResolvedTarget,
259 src: [:0]const u8,
260 result: [:0]const u8,
261) void {
262 ctx.addObj(name, target).addTransform(src, result);
263}
264
265/// Adds a test case that compiles the Zig given in `src` to ZIR and tests
266/// the ZIR against `result`
267pub fn transform(
268 ctx: *Cases,
269 name: []const u8,
270 target: std.Build.ResolvedTarget,
271 src: [:0]const u8,
272 result: [:0]const u8,
273) void {
274 ctx.addTransform(name, target, src, result);
275}
276
277pub fn addError(
278 ctx: *Cases,
279 name: []const u8,
280 target: std.Build.ResolvedTarget,
281 src: [:0]const u8,
282 expected_errors: []const []const u8,
283) void {
284 ctx.addObj(name, target).addError(src, expected_errors);
285}
286
287/// Adds a test case that ensures that the Zig given in `src` fails to
288/// compile for the expected reasons, given in sequential order in
289/// `expected_errors` in the form `:line:column: error: message`.
290pub fn compileError(
291 ctx: *Cases,
292 name: []const u8,
293 target: std.Build.ResolvedTarget,
294 src: [:0]const u8,
295 expected_errors: []const []const u8,
296) void {
297 ctx.addError(name, target, src, expected_errors);
298}
299
300/// Adds a test case that asserts that the Zig given in `src` compiles
301/// without any errors.
302pub fn addCompile(
303 ctx: *Cases,
304 name: []const u8,
305 target: std.Build.ResolvedTarget,
306 src: [:0]const u8,
307) void {
308 ctx.addObj(name, target).addCompile(src);
309}
310
311/// Adds a test for each file in the provided directory. Recurses nested directories.
312///
313/// Each file should include a test manifest as a contiguous block of comments at
314/// the end of the file. The first line should be the test type, followed by a set of
315/// key-value config values, followed by a blank line, then the expected output.
316pub fn addFromDir(ctx: *Cases, dir: std.fs.Dir, b: *std.Build) void {
317 var current_file: []const u8 = "none";
318 ctx.addFromDirInner(dir, ¤t_file, b) catch |err| {
319 std.debug.panicExtra(
320 @returnAddress(),
321 "test harness failed to process file '{s}': {s}\n",
322 .{ current_file, @errorName(err) },
323 );
324 };
325}
326
327fn addFromDirInner(
328 ctx: *Cases,
329 iterable_dir: std.fs.Dir,
330 /// This is kept up to date with the currently being processed file so
331 /// that if any errors occur the caller knows it happened during this file.
332 current_file: *[]const u8,
333 b: *std.Build,
334) !void {
335 var it = try iterable_dir.walk(ctx.arena);
336 var filenames: ArrayList([]const u8) = .empty;
337
338 while (try it.next()) |entry| {
339 if (entry.kind != .file) continue;
340
341 // Ignore stuff such as .swp files
342 if (!knownFileExtension(entry.basename)) continue;
343 try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
344 }
345
346 for (filenames.items) |filename| {
347 current_file.* = filename;
348
349 const max_file_size = 10 * 1024 * 1024;
350 const src = try iterable_dir.readFileAllocOptions(filename, ctx.arena, .limited(max_file_size), .@"1", 0);
351
352 // Parse the manifest
353 var manifest = try TestManifest.parse(ctx.arena, src);
354
355 const backends = try manifest.getConfigForKeyAlloc(ctx.arena, "backend", Backend);
356 const targets = try manifest.getConfigForKeyAlloc(ctx.arena, "target", std.Target.Query);
357 const is_test = try manifest.getConfigForKeyAssertSingle("is_test", bool);
358 const link_libc = try manifest.getConfigForKeyAssertSingle("link_libc", bool);
359 const output_mode = try manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode);
360 const pic = try manifest.getConfigForKeyAssertSingle("pic", ?bool);
361 const pie = try manifest.getConfigForKeyAssertSingle("pie", ?bool);
362 const emit_asm = try manifest.getConfigForKeyAssertSingle("emit_asm", bool);
363 const emit_bin = try manifest.getConfigForKeyAssertSingle("emit_bin", bool);
364 const imports = try manifest.getConfigForKeyAlloc(ctx.arena, "imports", []const u8);
365
366 var cases = std.array_list.Managed(usize).init(ctx.arena);
367
368 // Cross-product to get all possible test combinations
369 for (targets) |target_query| {
370 const resolved_target = b.resolveTargetQuery(target_query);
371 const target = &resolved_target.result;
372 for (backends) |backend| {
373 if (backend == .selfhosted and target.cpu.arch == .wasm32) {
374 // https://github.com/ziglang/zig/issues/25684
375 continue;
376 }
377 if (backend == .selfhosted and
378 target.cpu.arch != .aarch64 and target.cpu.arch != .wasm32 and target.cpu.arch != .x86_64 and target.cpu.arch != .spirv64)
379 {
380 // Other backends don't support new liveness format
381 continue;
382 }
383 if (backend == .selfhosted and target.os.tag == .macos and
384 target.cpu.arch == .x86_64 and builtin.cpu.arch == .aarch64)
385 {
386 // Rosetta has issues with ZLD
387 continue;
388 }
389
390 const next = ctx.cases.items.len;
391 try ctx.cases.append(.{
392 .name = try caseNameFromPath(ctx.arena, filename),
393 .import_path = std.fs.path.dirname(filename),
394 .backend = backend,
395 .files = .init(ctx.arena),
396 .case = null,
397 .emit_asm = emit_asm,
398 .emit_bin = emit_bin,
399 .is_test = is_test,
400 .output_mode = output_mode,
401 .link_libc = link_libc,
402 .pic = pic,
403 .pie = pie,
404 .deps = std.array_list.Managed(DepModule).init(ctx.cases.allocator),
405 .imports = imports,
406 .target = resolved_target,
407 });
408 try cases.append(next);
409 }
410 }
411
412 for (cases.items) |case_index| {
413 const case = &ctx.cases.items[case_index];
414 switch (manifest.type) {
415 .compile => {
416 case.addCompile(src);
417 },
418 .@"error" => {
419 const errors = try manifest.trailingLines(ctx.arena);
420 case.addError(src, errors);
421 },
422 .run => {
423 const output = try manifest.trailingSplit(ctx.arena);
424 case.addCompareOutput(src, output);
425 },
426 .translate_c => @panic("c_frontend specified for compile case"),
427 .run_translated_c => @panic("c_frontend specified for compile case"),
428 .cli => @panic("TODO cli tests"),
429 }
430 }
431 }
432}
433
434pub fn init(gpa: Allocator, arena: Allocator) Cases {
435 return .{
436 .gpa = gpa,
437 .cases = .init(gpa),
438 .arena = arena,
439 };
440}
441
442pub const CaseTestOptions = struct {
443 test_filters: []const []const u8,
444 test_target_filters: []const []const u8,
445 skip_compile_errors: bool,
446 skip_non_native: bool,
447 skip_freebsd: bool,
448 skip_netbsd: bool,
449 skip_windows: bool,
450 skip_darwin: bool,
451 skip_linux: bool,
452 skip_llvm: bool,
453 skip_libc: bool,
454};
455
456pub fn lowerToBuildSteps(
457 self: *Cases,
458 b: *std.Build,
459 parent_step: *std.Build.Step,
460 options: CaseTestOptions,
461) void {
462 const host = b.resolveTargetQuery(.{});
463 const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM");
464
465 for (self.cases.items) |case| {
466 for (options.test_filters) |test_filter| {
467 if (std.mem.indexOf(u8, case.name, test_filter)) |_| break;
468 } else if (options.test_filters.len > 0) continue;
469
470 if (case.case.? == .Error and options.skip_compile_errors) continue;
471
472 if (options.skip_non_native and !case.target.query.isNative())
473 continue;
474
475 if (options.skip_freebsd and case.target.query.os_tag == .freebsd) continue;
476 if (options.skip_netbsd and case.target.query.os_tag == .netbsd) continue;
477 if (options.skip_windows and case.target.query.os_tag == .windows) continue;
478 if (options.skip_darwin and case.target.query.os_tag != null and case.target.query.os_tag.?.isDarwin()) continue;
479 if (options.skip_linux and case.target.query.os_tag == .linux) continue;
480
481 const would_use_llvm = @import("../tests.zig").wouldUseLlvm(
482 switch (case.backend) {
483 .auto => null,
484 .selfhosted => false,
485 .llvm => true,
486 },
487 case.target.query,
488 case.optimize_mode,
489 );
490 if (options.skip_llvm and would_use_llvm) continue;
491
492 const triple_txt = case.target.query.zigTriple(b.allocator) catch @panic("OOM");
493
494 if (options.test_target_filters.len > 0) {
495 for (options.test_target_filters) |filter| {
496 if (std.mem.indexOf(u8, triple_txt, filter) != null) break;
497 } else continue;
498 }
499
500 if (options.skip_libc and case.link_libc)
501 continue;
502
503 const writefiles = b.addWriteFiles();
504 var file_sources = std.StringHashMap(std.Build.LazyPath).init(b.allocator);
505 defer file_sources.deinit();
506 const first_file = case.files.items[0];
507 const root_source_file = writefiles.add(first_file.path, first_file.src);
508 file_sources.put(first_file.path, root_source_file) catch @panic("OOM");
509 for (case.files.items[1..]) |file| {
510 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM");
511 }
512
513 for (case.imports) |import_rel| {
514 const import_abs = std.fs.path.join(b.allocator, &.{
515 cases_dir_path,
516 case.import_path orelse @panic("import_path not set"),
517 import_rel,
518 }) catch @panic("OOM");
519 _ = writefiles.addCopyFile(.{ .cwd_relative = import_abs }, import_rel);
520 }
521
522 const mod = b.createModule(.{
523 .root_source_file = root_source_file,
524 .target = case.target,
525 .optimize = case.optimize_mode,
526 });
527
528 if (case.link_libc) mod.link_libc = true;
529 if (case.pic) |pic| mod.pic = pic;
530 for (case.deps.items) |dep| {
531 mod.addAnonymousImport(dep.name, .{
532 .root_source_file = file_sources.get(dep.path).?,
533 });
534 }
535
536 const artifact = if (case.is_test) b.addTest(.{
537 .name = case.name,
538 .root_module = mod,
539 }) else switch (case.output_mode) {
540 .Obj => b.addObject(.{
541 .name = case.name,
542 .root_module = mod,
543 }),
544 .Lib => b.addLibrary(.{
545 .linkage = .static,
546 .name = case.name,
547 .root_module = mod,
548 }),
549 .Exe => b.addExecutable(.{
550 .name = case.name,
551 .root_module = mod,
552 }),
553 };
554
555 if (case.pie) |pie| artifact.pie = pie;
556
557 switch (case.backend) {
558 .auto => {},
559 .selfhosted => {
560 artifact.use_llvm = false;
561 artifact.use_lld = false;
562 },
563 .llvm => {
564 artifact.use_llvm = true;
565 },
566 }
567
568 switch (case.case.?) {
569 .Compile => {
570 // Force the assembly/binary to be emitted if requested.
571 if (case.emit_asm) {
572 _ = artifact.getEmittedAsm();
573 }
574 if (case.emit_bin) {
575 _ = artifact.getEmittedBin();
576 }
577 parent_step.dependOn(&artifact.step);
578 },
579 .CompareObjectFile => |expected_output| {
580 const check = b.addCheckFile(artifact.getEmittedBin(), .{
581 .expected_exact = expected_output,
582 });
583
584 parent_step.dependOn(&check.step);
585 },
586 .Error => |expected_msgs| {
587 assert(expected_msgs.len != 0);
588 artifact.expect_errors = .{ .exact = expected_msgs };
589 parent_step.dependOn(&artifact.step);
590 },
591 .Execution => |expected_stdout| no_exec: {
592 const run = if (case.target.result.ofmt == .c) run_step: {
593 if (getExternalExecutor(&host.result, &case.target.result, .{ .link_libc = true }) != .native) {
594 // We wouldn't be able to run the compiled C code.
595 break :no_exec;
596 }
597 const run_c = b.addSystemCommand(&.{
598 b.graph.zig_exe,
599 "run",
600 "-cflags",
601 "-Ilib",
602 "-std=c99",
603 "-pedantic",
604 "-Werror",
605 "-Wno-dollar-in-identifier-extension",
606 "-Wno-incompatible-library-redeclaration", // https://github.com/ziglang/zig/issues/875
607 "-Wno-incompatible-pointer-types",
608 "-Wno-overlength-strings",
609 "--",
610 "-lc",
611 "-target",
612 triple_txt,
613 });
614 run_c.addArtifactArg(artifact);
615 break :run_step run_c;
616 } else b.addRunArtifact(artifact);
617 run.skip_foreign_checks = true;
618 if (!case.is_test) {
619 run.expectStdOutEqual(expected_stdout);
620 }
621 parent_step.dependOn(&run.step);
622 },
623 .Header => @panic("TODO"),
624 }
625 }
626}
627
628/// Default config values for known test manifest key-value pairings.
629/// Currently handled defaults are:
630/// * backend
631/// * target
632/// * output_mode
633/// * is_test
634const TestManifestConfigDefaults = struct {
635 /// Asserts if the key doesn't exist - yep, it's an oversight alright.
636 fn get(@"type": TestManifest.Type, key: []const u8) []const u8 {
637 if (std.mem.eql(u8, key, "backend")) {
638 return "auto";
639 } else if (std.mem.eql(u8, key, "target")) {
640 if (@"type" == .@"error" or @"type" == .translate_c or @"type" == .run_translated_c) {
641 return "native";
642 }
643 return comptime blk: {
644 var defaults: []const u8 = "";
645 // TODO should we only return "mainstream" targets by default here?
646 // TODO we should also specify ABIs explicitly as the backends are
647 // getting more and more complete
648 // Linux
649 for (&[_][]const u8{ "x86_64", "arm", "aarch64" }) |arch| {
650 defaults = defaults ++ arch ++ "-linux" ++ ",";
651 }
652 // macOS
653 for (&[_][]const u8{ "x86_64", "aarch64" }) |arch| {
654 defaults = defaults ++ arch ++ "-macos" ++ ",";
655 }
656 // Windows
657 defaults = defaults ++ "x86_64-windows" ++ ",";
658 // Wasm
659 defaults = defaults ++ "wasm32-wasi";
660 break :blk defaults;
661 };
662 } else if (std.mem.eql(u8, key, "output_mode")) {
663 return switch (@"type") {
664 .@"error" => "Obj",
665 .run => "Exe",
666 .compile => "Obj",
667 .translate_c => "Obj",
668 .run_translated_c => "Obj",
669 .cli => @panic("TODO test harness for CLI tests"),
670 };
671 } else if (std.mem.eql(u8, key, "emit_asm")) {
672 return "false";
673 } else if (std.mem.eql(u8, key, "emit_bin")) {
674 return "true";
675 } else if (std.mem.eql(u8, key, "is_test")) {
676 return "false";
677 } else if (std.mem.eql(u8, key, "link_libc")) {
678 return "false";
679 } else if (std.mem.eql(u8, key, "c_frontend")) {
680 return "clang";
681 } else if (std.mem.eql(u8, key, "pic")) {
682 return "null";
683 } else if (std.mem.eql(u8, key, "pie")) {
684 return "null";
685 } else if (std.mem.eql(u8, key, "imports")) {
686 return "";
687 } else unreachable;
688 }
689};
690
691/// Manifest syntax example:
692/// (see https://github.com/ziglang/zig/issues/11288)
693///
694/// error
695/// backend=selfhosted,llvm
696/// output_mode=exe
697///
698/// :3:19: error: foo
699///
700/// run
701/// target=x86_64-linux,aarch64-macos
702///
703/// I am expected stdout! Hello!
704///
705/// cli
706///
707/// build test
708const TestManifest = struct {
709 type: Type,
710 config_map: std.StringHashMap([]const u8),
711 trailing_bytes: []const u8 = "",
712
713 const valid_keys = std.StaticStringMap(void).initComptime(.{
714 .{ "emit_asm", {} },
715 .{ "emit_bin", {} },
716 .{ "is_test", {} },
717 .{ "output_mode", {} },
718 .{ "target", {} },
719 .{ "c_frontend", {} },
720 .{ "link_libc", {} },
721 .{ "backend", {} },
722 .{ "pic", {} },
723 .{ "pie", {} },
724 .{ "imports", {} },
725 });
726
727 const Type = enum {
728 @"error",
729 run,
730 cli,
731 compile,
732 translate_c,
733 run_translated_c,
734 };
735
736 const TrailingIterator = struct {
737 inner: std.mem.TokenIterator(u8, .any),
738
739 fn next(self: *TrailingIterator) ?[]const u8 {
740 const next_inner = self.inner.next() orelse return null;
741 return if (next_inner.len == 2) "" else std.mem.trimEnd(u8, next_inner[3..], " \t");
742 }
743 };
744
745 fn ConfigValueIterator(comptime T: type) type {
746 return struct {
747 inner: std.mem.TokenIterator(u8, .scalar),
748
749 fn next(self: *@This()) !?T {
750 const next_raw = self.inner.next() orelse return null;
751 const parseFn = getDefaultParser(T);
752 return try parseFn(next_raw);
753 }
754 };
755 }
756
757 fn parse(arena: Allocator, bytes: []const u8) !TestManifest {
758 // The manifest is the last contiguous block of comments in the file
759 // We scan for the beginning by searching backward for the first non-empty line that does not start with "//"
760 var start: ?usize = null;
761 var end: usize = bytes.len;
762 if (bytes.len > 0) {
763 var cursor: usize = bytes.len - 1;
764 while (true) {
765 // Move to beginning of line
766 while (cursor > 0 and bytes[cursor - 1] != '\n') cursor -= 1;
767
768 if (std.mem.startsWith(u8, bytes[cursor..], "//")) {
769 start = cursor; // Contiguous comment line, include in manifest
770 } else {
771 if (start != null) break; // Encountered non-comment line, end of manifest
772
773 // We ignore all-whitespace lines following the comment block, but anything else
774 // means that there is no manifest present.
775 if (std.mem.trim(u8, bytes[cursor..end], " \r\n\t").len == 0) {
776 end = cursor;
777 } else break; // If it's not whitespace, there is no manifest
778 }
779
780 // Move to previous line
781 if (cursor != 0) cursor -= 1 else break;
782 }
783 }
784
785 const actual_start = start orelse return error.MissingTestManifest;
786 const manifest_bytes = bytes[actual_start..end];
787
788 var it = std.mem.tokenizeAny(u8, manifest_bytes, "\r\n");
789
790 // First line is the test type
791 const tt: Type = blk: {
792 const line = it.next() orelse return error.MissingTestCaseType;
793 const raw = std.mem.trim(u8, line[2..], " \t");
794 if (std.mem.eql(u8, raw, "error")) {
795 break :blk .@"error";
796 } else if (std.mem.eql(u8, raw, "run")) {
797 break :blk .run;
798 } else if (std.mem.eql(u8, raw, "cli")) {
799 break :blk .cli;
800 } else if (std.mem.eql(u8, raw, "compile")) {
801 break :blk .compile;
802 } else if (std.mem.eql(u8, raw, "translate-c")) {
803 break :blk .translate_c;
804 } else if (std.mem.eql(u8, raw, "run-translated-c")) {
805 break :blk .run_translated_c;
806 } else {
807 std.log.warn("unknown test case type requested: {s}", .{raw});
808 return error.UnknownTestCaseType;
809 }
810 };
811
812 var manifest: TestManifest = .{
813 .type = tt,
814 .config_map = std.StringHashMap([]const u8).init(arena),
815 };
816
817 // Any subsequent line until a blank comment line is key=value(s) pair
818 while (it.next()) |line| {
819 const trimmed = std.mem.trim(u8, line[2..], " \t");
820 if (trimmed.len == 0) break;
821
822 // Parse key=value(s)
823 var kv_it = std.mem.splitScalar(u8, trimmed, '=');
824 const key = kv_it.first();
825 if (!valid_keys.has(key)) {
826 return error.InvalidKey;
827 }
828 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig);
829 }
830
831 // Finally, trailing is expected output
832 manifest.trailing_bytes = manifest_bytes[it.index..];
833
834 return manifest;
835 }
836
837 fn getConfigForKey(
838 self: TestManifest,
839 key: []const u8,
840 comptime T: type,
841 ) ConfigValueIterator(T) {
842 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.type, key);
843 return ConfigValueIterator(T){
844 .inner = std.mem.tokenizeScalar(u8, bytes, ','),
845 };
846 }
847
848 fn getConfigForKeyAlloc(
849 self: TestManifest,
850 allocator: Allocator,
851 key: []const u8,
852 comptime T: type,
853 ) ![]const T {
854 var out = std.array_list.Managed(T).init(allocator);
855 defer out.deinit();
856 var it = self.getConfigForKey(key, T);
857 while (try it.next()) |item| {
858 try out.append(item);
859 }
860 return try out.toOwnedSlice();
861 }
862
863 fn getConfigForKeyAssertSingle(self: TestManifest, key: []const u8, comptime T: type) !T {
864 var it = self.getConfigForKey(key, T);
865 const res = (try it.next()) orelse unreachable;
866 assert((try it.next()) == null);
867 return res;
868 }
869
870 fn trailing(self: TestManifest) TrailingIterator {
871 return .{
872 .inner = std.mem.tokenizeAny(u8, self.trailing_bytes, "\r\n"),
873 };
874 }
875
876 fn trailingSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const u8 {
877 var out = std.array_list.Managed(u8).init(allocator);
878 defer out.deinit();
879 var trailing_it = self.trailing();
880 while (trailing_it.next()) |line| {
881 try out.appendSlice(line);
882 try out.append('\n');
883 }
884 if (out.items.len > 0) {
885 try out.resize(out.items.len - 1);
886 }
887 return try out.toOwnedSlice();
888 }
889
890 fn trailingLines(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 {
891 var out = std.array_list.Managed([]const u8).init(allocator);
892 defer out.deinit();
893 var it = self.trailing();
894 while (it.next()) |line| {
895 try out.append(line);
896 }
897 return try out.toOwnedSlice();
898 }
899
900 fn trailingLinesSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 {
901 // Collect output lines split by empty lines
902 var out = std.array_list.Managed([]const u8).init(allocator);
903 defer out.deinit();
904 var buf = std.array_list.Managed(u8).init(allocator);
905 defer buf.deinit();
906 var it = self.trailing();
907 while (it.next()) |line| {
908 if (line.len == 0) {
909 if (buf.items.len != 0) {
910 try out.append(try buf.toOwnedSlice());
911 buf.items.len = 0;
912 }
913 continue;
914 }
915 try buf.appendSlice(line);
916 try buf.append('\n');
917 }
918 try out.append(try buf.toOwnedSlice());
919 return try out.toOwnedSlice();
920 }
921
922 fn ParseFn(comptime T: type) type {
923 return fn ([]const u8) anyerror!T;
924 }
925
926 fn getDefaultParser(comptime T: type) ParseFn(T) {
927 if (T == std.Target.Query) return struct {
928 fn parse(str: []const u8) anyerror!T {
929 return std.Target.Query.parse(.{ .arch_os_abi = str });
930 }
931 }.parse;
932
933 switch (@typeInfo(T)) {
934 .int => return struct {
935 fn parse(str: []const u8) anyerror!T {
936 return try std.fmt.parseInt(T, str, 0);
937 }
938 }.parse,
939 .bool => return struct {
940 fn parse(str: []const u8) anyerror!T {
941 if (std.mem.eql(u8, str, "true")) return true;
942 if (std.mem.eql(u8, str, "false")) return false;
943 std.debug.print("{s}\n", .{str});
944 return error.InvalidBool;
945 }
946 }.parse,
947 .@"enum" => return struct {
948 fn parse(str: []const u8) anyerror!T {
949 return std.meta.stringToEnum(T, str) orelse {
950 std.log.err("unknown enum variant for {s}: {s}", .{ @typeName(T), str });
951 return error.UnknownEnumVariant;
952 };
953 }
954 }.parse,
955 .optional => |o| return struct {
956 fn parse(str: []const u8) anyerror!T {
957 if (std.mem.eql(u8, str, "null")) return null;
958 return try getDefaultParser(o.child)(str);
959 }
960 }.parse,
961 .@"struct" => @compileError("no default parser for " ++ @typeName(T)),
962 .pointer => {
963 if (T == []const u8) {
964 return struct {
965 fn parse(str: []const u8) anyerror!T {
966 return str;
967 }
968 }.parse;
969 } else {
970 @compileError("no default parser for " ++ @typeName(T));
971 }
972 },
973 else => @compileError("no default parser for " ++ @typeName(T)),
974 }
975 }
976};
977
978fn knownFileExtension(filename: []const u8) bool {
979 // List taken from `Compilation.classifyFileExt` in the compiler.
980 for ([_][]const u8{
981 ".c", ".C", ".cc", ".cpp",
982 ".cxx", ".stub", ".m", ".mm",
983 ".ll", ".bc", ".s", ".S",
984 ".h", ".zig", ".so", ".dll",
985 ".dylib", ".tbd", ".a", ".lib",
986 ".o", ".obj", ".cu", ".def",
987 ".rc", ".res", ".manifest",
988 }) |ext| {
989 if (std.mem.endsWith(u8, filename, ext)) return true;
990 }
991 // Final check for .so.X, .so.X.Y, .so.X.Y.Z.
992 // From `Compilation.hasSharedLibraryExt`.
993 var it = std.mem.splitScalar(u8, filename, '.');
994 _ = it.first();
995 var so_txt = it.next() orelse return false;
996 while (!std.mem.eql(u8, so_txt, "so")) {
997 so_txt = it.next() orelse return false;
998 }
999 const n1 = it.next() orelse return false;
1000 const n2 = it.next();
1001 const n3 = it.next();
1002 _ = std.fmt.parseInt(u32, n1, 10) catch return false;
1003 if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false;
1004 if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false;
1005 if (it.next() != null) return false;
1006 return false;
1007}
1008
1009/// `path` is a path relative to the root case directory.
1010/// e.g. `compile_errors/undeclared_identifier.zig`
1011/// The case name is computed by removing the extension and substituting path separators for dots.
1012/// e.g. `compile_errors.undeclared_identifier`
1013/// Including the directory components makes `-Dtest-filter` more useful, because you can filter
1014/// based on subdirectory; e.g. `-Dtest-filter=compile_errors` to run the compile error tets.
1015fn caseNameFromPath(arena: Allocator, path: []const u8) Allocator.Error![]const u8 {
1016 const ext_len = std.fs.path.extension(path).len;
1017 const path_sans_ext = path[0 .. path.len - ext_len];
1018 const result = try arena.dupe(u8, path_sans_ext);
1019 std.mem.replaceScalar(u8, result, std.fs.path.sep, '.');
1020 return result;
1021}