Commit 53d1282f0c

kcbanner <kcbanner@gmail.com>
2023-04-28 06:20:25
build: add --skip-oom-steps
Specifying this argument causes steps that specify a max_rss that exceeds the total max_rss value of the runner to be skipped, instead of failing.
1 parent e011c31
ci/aarch64-macos-release.sh
@@ -46,6 +46,7 @@ $HOME/local/bin/ninja install
 
 stage3-release/bin/zig build test docs \
   --zig-lib-dir "$(pwd)/../lib" \
+  --skip-oom-steps \
   -Denable-macos-sdk \
   -Dstatic-llvm \
   -Dskip-non-native \
ci/x86_64-windows-debug.ps1
@@ -57,6 +57,7 @@ Write-Output "Main test suite..."
 & "stage3-debug\bin\zig.exe" build test docs `
   --zig-lib-dir "$ZIG_LIB_DIR" `
   --search-prefix "$PREFIX_PATH" `
+  --skip-oom-steps `
   -Dstatic-llvm `
   -Dskip-non-native `
   -Denable-symlinks-windows
ci/x86_64-windows-release.ps1
@@ -57,6 +57,7 @@ Write-Output "Main test suite..."
 & "stage3-release\bin\zig.exe" build test docs `
   --zig-lib-dir "$ZIG_LIB_DIR" `
   --search-prefix "$PREFIX_PATH" `
+  --skip-oom-steps `
   -Dstatic-llvm `
   -Dskip-non-native `
   -Denable-symlinks-windows
lib/std/Build/Step.zig
@@ -70,6 +70,9 @@ pub const State = enum {
     /// This state indicates that the step did not complete, however, it also did not fail,
     /// and it is safe to continue executing its dependencies.
     skipped,
+    /// This step was skipped because it specified a max_rss that exceeded the runner's maximum.
+    /// It is not safe to run its dependencies.
+    skipped_oom,
 };
 
 pub const Id = enum {
lib/build_runner.zig
@@ -93,6 +93,7 @@ pub fn main() !void {
     var dir_list = std.Build.DirList{};
     var summary: ?Summary = null;
     var max_rss: usize = 0;
+    var skip_oom_steps: bool = false;
     var color: Color = .auto;
 
     const stderr_stream = io.getStdErr().writer();
@@ -158,6 +159,8 @@ pub fn main() !void {
                     });
                     process.exit(1);
                 };
+            } else if (mem.eql(u8, arg, "--skip-oom-steps")) {
+                skip_oom_steps = true;
             } else if (mem.eql(u8, arg, "--search-prefix")) {
                 const search_prefix = nextArg(args, &arg_idx) orelse {
                     std.debug.print("Expected argument after {s}\n\n", .{arg});
@@ -305,6 +308,7 @@ pub fn main() !void {
         .max_rss = max_rss,
         .max_rss_is_default = false,
         .max_rss_mutex = .{},
+        .skip_oom_steps = skip_oom_steps,
         .memory_blocked_steps = std.ArrayList(*Step).init(arena),
 
         .claimed_rss = 0,
@@ -335,6 +339,7 @@ const Run = struct {
     max_rss: usize,
     max_rss_is_default: bool,
     max_rss_mutex: std.Thread.Mutex,
+    skip_oom_steps: bool,
     memory_blocked_steps: std.ArrayList(*Step),
 
     claimed_rss: usize,
@@ -383,10 +388,14 @@ fn runStepNames(
         for (step_stack.keys()) |s| {
             if (s.max_rss == 0) continue;
             if (s.max_rss > run.max_rss) {
-                std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{
-                    s.owner.dep_prefix, s.name, s.max_rss, run.max_rss,
-                });
-                any_problems = true;
+                if (run.skip_oom_steps) {
+                    s.state = .skipped_oom;
+                } else {
+                    std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{
+                        s.owner.dep_prefix, s.name, s.max_rss, run.max_rss,
+                    });
+                    any_problems = true;
+                }
             }
         }
         if (any_problems) {
@@ -416,6 +425,7 @@ fn runStepNames(
         const steps_slice = step_stack.keys();
         for (0..steps_slice.len) |i| {
             const step = steps_slice[steps_slice.len - i - 1];
+            if (step.state == .skipped_oom) continue;
 
             wait_group.start();
             thread_pool.spawn(workerMakeOneStep, .{
@@ -461,7 +471,7 @@ fn runStepNames(
             },
             .dependency_failure => pending_count += 1,
             .success => success_count += 1,
-            .skipped => skipped_count += 1,
+            .skipped, .skipped_oom => skipped_count += 1,
             .failure => {
                 failure_count += 1;
                 const compile_errors_len = s.result_error_bundle.errorMessageCount();
@@ -506,7 +516,7 @@ fn runStepNames(
         var print_node: PrintNode = .{ .parent = null };
         if (step_names.len == 0) {
             print_node.last = true;
-            printTreeStep(b, b.default_step, stderr, ttyconf, &print_node, &step_stack, failures_only) catch {};
+            printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack, failures_only) catch {};
         } else {
             const last_index = if (!failures_only) b.top_level_steps.count() else blk: {
                 var i: usize = step_names.len;
@@ -519,7 +529,7 @@ fn runStepNames(
             for (step_names, 0..) |step_name, i| {
                 const tls = b.top_level_steps.get(step_name).?;
                 print_node.last = i + 1 == last_index;
-                printTreeStep(b, &tls.step, stderr, ttyconf, &print_node, &step_stack, failures_only) catch {};
+                printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack, failures_only) catch {};
             }
         }
     }
@@ -567,6 +577,7 @@ fn printPrefix(node: *PrintNode, stderr: std.fs.File, ttyconf: std.io.tty.Config
 fn printTreeStep(
     b: *std.Build,
     s: *Step,
+    run: *const Run,
     stderr: std.fs.File,
     ttyconf: std.io.tty.Config,
     parent_node: *PrintNode,
@@ -654,13 +665,18 @@ fn printTreeStep(
                 }
                 try stderr.writeAll("\n");
             },
-
-            .skipped => {
+            .skipped, .skipped_oom => |skip| {
                 try ttyconf.setColor(stderr, .yellow);
-                try stderr.writeAll(" skipped\n");
+                try stderr.writeAll(" skipped");
+                if (skip == .skipped_oom) {
+                    try stderr.writeAll(" (not enough memory)");
+                    try ttyconf.setColor(stderr, .dim);
+                    try stderr.writer().print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss });
+                    try ttyconf.setColor(stderr, .yellow);
+                }
+                try stderr.writeAll("\n");
                 try ttyconf.setColor(stderr, .reset);
             },
-
             .failure => {
                 if (s.result_error_bundle.errorMessageCount() > 0) {
                     try ttyconf.setColor(stderr, .red);
@@ -718,7 +734,7 @@ fn printTreeStep(
                 .parent = parent_node,
                 .last = i == last_index,
             };
-            try printTreeStep(b, dep, stderr, ttyconf, &print_node, step_stack, failures_only);
+            try printTreeStep(b, dep, run, stderr, ttyconf, &print_node, step_stack, failures_only);
         }
     } else {
         if (s.dependencies.items.len == 0) {
@@ -767,6 +783,7 @@ fn checkForDependencyLoop(
         .success => unreachable,
         .failure => unreachable,
         .skipped => unreachable,
+        .skipped_oom => unreachable,
     }
 }
 
@@ -786,7 +803,7 @@ fn workerMakeOneStep(
     for (s.dependencies.items) |dep| {
         switch (@atomicLoad(Step.State, &dep.state, .SeqCst)) {
             .success, .skipped => continue,
-            .failure, .dependency_failure => {
+            .failure, .dependency_failure, .skipped_oom => {
                 @atomicStore(Step.State, &s.state, .dependency_failure, .SeqCst);
                 return;
             },
@@ -979,6 +996,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
         \\    none                       Do not print the build summary
         \\  -j<N>                        Limit concurrent jobs (default is to use all CPU cores)
         \\  --maxrss <bytes>             Limit memory usage (default is to use available memory)
+        \\  --skip-oom-steps             Instead of failing, skip steps that would exceed --maxrss
         \\
         \\Project-Specific Options:
         \\
test/tests.zig
@@ -1052,6 +1052,7 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
                 .link_libc = test_target.link_libc,
                 .target = altered_target,
                 .zig_lib_dir = .{ .path = "lib" },
+                .max_rss = if (mem.eql(u8, options.name, "std")) 11199083316 else 0,
             });
             compile_c.addCSourceFile(.{
                 .file = these_tests.getEmittedBin(),