Commit 5cd548e530

Andrew Kelley <andrew@ziglang.org>
2022-06-17 05:23:22
Compilation: multi-thread compiler-rt
compiler_rt_lib and compiler_rt_obj are extracted from the generic JobQueue into simple boolean flags, and then handled explicitly inside performAllTheWork(). Introduced generic handling of allocation failure and made setMiscFailure not return a possible error. Building the compiler-rt static library now takes advantage of Compilation's ThreadPool. This introduced a problem, however, because now each of the object files of compiler-rt all perform AstGen for the full standard library and compiler-rt files. Even though all of them end up being cache hits except for the first ones, this is wasteful - O(N*M) where N is number of compilation units inside compiler-rt and M is the number of .zig files in the standard library and compiler-rt combined. More importantly, however, it causes a deadlock, because each thread interacts with a file system lock for doing AstGen on files, and threads end up waiting for each other. This will need to be handled with a process-level file caching system, or some other creative solution.
1 parent b4f3e69
src/Compilation.zig
@@ -93,6 +93,9 @@ unwind_tables: bool,
 test_evented_io: bool,
 debug_compiler_runtime_libs: bool,
 debug_compile_errors: bool,
+job_queued_compiler_rt_lib: bool = false,
+job_queued_compiler_rt_obj: bool = false,
+alloc_failure_occurred: bool = false,
 
 c_source_files: []const CSourceFile,
 clang_argv: []const []const u8,
@@ -130,11 +133,11 @@ libssp_static_lib: ?CRTFile = null,
 /// Populated when we build the libc static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
 libc_static_lib: ?CRTFile = null,
-/// Populated when we build the libcompiler_rt static library. A Job to build this is placed in the queue
-/// and resolved before calling linker.flush().
+/// Populated when we build the libcompiler_rt static library. A Job to build this is indicated
+/// by setting `job_queued_compiler_rt_lib` and resolved before calling linker.flush().
 compiler_rt_lib: ?CRTFile = null,
-/// Populated when we build the compiler_rt_obj object. A Job to build this is placed in the queue
-/// and resolved before calling linker.flush().
+/// Populated when we build the compiler_rt_obj object. A Job to build this is indicated
+/// by setting `job_queued_compiler_rt_obj` and resolved before calling linker.flush().
 compiler_rt_obj: ?CRTFile = null,
 
 glibc_so_files: ?glibc.BuiltSharedObjects = null,
@@ -224,8 +227,6 @@ const Job = union(enum) {
     libcxxabi: void,
     libtsan: void,
     libssp: void,
-    compiler_rt_lib: void,
-    compiler_rt_obj: void,
     /// needed when not linking libc and using LLVM for code generation because it generates
     /// calls to, for example, memcpy and memset.
     zig_libc: void,
@@ -1925,13 +1926,13 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         if (comp.bin_file.options.include_compiler_rt and capable_of_building_compiler_rt) {
             if (is_exe_or_dyn_lib) {
                 log.debug("queuing a job to build compiler_rt_lib", .{});
-                try comp.work_queue.writeItem(.{ .compiler_rt_lib = {} });
+                comp.job_queued_compiler_rt_lib = true;
             } else if (options.output_mode != .Obj) {
                 log.debug("queuing a job to build compiler_rt_obj", .{});
                 // If build-obj with -fcompiler-rt is requested, that is handled specially
                 // elsewhere. In this case we are making a static library, so we ask
                 // for a compiler-rt object to put in it.
-                try comp.work_queue.writeItem(.{ .compiler_rt_obj = {} });
+                comp.job_queued_compiler_rt_obj = true;
             }
         }
         if (needs_c_symbols) {
@@ -2021,6 +2022,7 @@ pub fn destroy(self: *Compilation) void {
 }
 
 pub fn clearMiscFailures(comp: *Compilation) void {
+    comp.alloc_failure_occurred = false;
     for (comp.misc_failures.values()) |*value| {
         value.deinit(comp.gpa);
     }
@@ -2533,8 +2535,10 @@ pub fn makeBinFileWritable(self: *Compilation) !void {
     return self.bin_file.makeWritable();
 }
 
+/// This function is temporally single-threaded.
 pub fn totalErrorCount(self: *Compilation) usize {
-    var total: usize = self.failed_c_objects.count() + self.misc_failures.count();
+    var total: usize = self.failed_c_objects.count() + self.misc_failures.count() +
+        @boolToInt(self.alloc_failure_occurred);
 
     if (self.bin_file.options.module) |module| {
         total += module.failed_exports.count();
@@ -2591,6 +2595,7 @@ pub fn totalErrorCount(self: *Compilation) usize {
     return total;
 }
 
+/// This function is temporally single-threaded.
 pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     var arena = std.heap.ArenaAllocator.init(self.gpa);
     errdefer arena.deinit();
@@ -2623,6 +2628,9 @@ pub fn getAllErrorsAlloc(self: *Compilation) !AllErrors {
     for (self.misc_failures.values()) |*value| {
         try AllErrors.addPlainWithChildren(&arena, &errors, value.msg, value.children);
     }
+    if (self.alloc_failure_occurred) {
+        try AllErrors.addPlain(&arena, &errors, "memory allocation failure");
+    }
     if (self.bin_file.options.module) |module| {
         {
             var it = module.failed_files.iterator();
@@ -2737,9 +2745,15 @@ pub fn performAllTheWork(
     var embed_file_prog_node = main_progress_node.start("Detect @embedFile updates", comp.embed_file_work_queue.count);
     defer embed_file_prog_node.end();
 
+    // +1 for the link step
+    var compiler_rt_prog_node = main_progress_node.start("compiler_rt", compiler_rt.sources.len + 1);
+    defer compiler_rt_prog_node.end();
+
     comp.work_queue_wait_group.reset();
     defer comp.work_queue_wait_group.wait();
 
+    const use_stage1 = build_options.is_stage1 and comp.bin_file.options.use_stage1;
+
     {
         const astgen_frame = tracy.namedFrame("astgen");
         defer astgen_frame.end();
@@ -2782,9 +2796,28 @@ pub fn performAllTheWork(
                 comp, c_object, &c_obj_prog_node, &comp.work_queue_wait_group,
             });
         }
+
+        if (comp.job_queued_compiler_rt_lib) {
+            comp.job_queued_compiler_rt_lib = false;
+
+            if (use_stage1) {
+                // stage1 LLVM backend uses the global context and thus cannot be used in
+                // a multi-threaded context.
+                buildCompilerRtOneShot(comp, .Lib, &comp.compiler_rt_lib);
+            } else {
+                comp.work_queue_wait_group.start();
+                try comp.thread_pool.spawn(workerBuildCompilerRtLib, .{
+                    comp, &compiler_rt_prog_node, &comp.work_queue_wait_group,
+                });
+            }
+        }
+
+        if (comp.job_queued_compiler_rt_obj) {
+            comp.job_queued_compiler_rt_obj = false;
+            buildCompilerRtOneShot(comp, .Obj, &comp.compiler_rt_obj);
+        }
     }
 
-    const use_stage1 = build_options.is_stage1 and comp.bin_file.options.use_stage1;
     if (!use_stage1) {
         const outdated_and_deleted_decls_frame = tracy.namedFrame("outdated_and_deleted_decls");
         defer outdated_and_deleted_decls_frame.end();
@@ -2997,7 +3030,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
             module.semaPkg(pkg) catch |err| switch (err) {
                 error.CurrentWorkingDirectoryUnlinked,
                 error.Unexpected,
-                => try comp.setMiscFailure(
+                => comp.lockAndSetMiscFailure(
                     .analyze_pkg,
                     "unexpected problem analyzing package '{s}'",
                     .{pkg.root_src_path},
@@ -3012,7 +3045,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             glibc.buildCRTFile(comp, crt_file) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(.glibc_crt_file, "unable to build glibc CRT file: {s}", .{
+                comp.lockAndSetMiscFailure(.glibc_crt_file, "unable to build glibc CRT file: {s}", .{
                     @errorName(err),
                 });
             };
@@ -3023,7 +3056,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             glibc.buildSharedObjects(comp) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .glibc_shared_objects,
                     "unable to build glibc shared objects: {s}",
                     .{@errorName(err)},
@@ -3036,7 +3069,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             musl.buildCRTFile(comp, crt_file) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .musl_crt_file,
                     "unable to build musl CRT file: {s}",
                     .{@errorName(err)},
@@ -3049,7 +3082,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             mingw.buildCRTFile(comp, crt_file) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .mingw_crt_file,
                     "unable to build mingw-w64 CRT file: {s}",
                     .{@errorName(err)},
@@ -3063,7 +3096,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
             const link_lib = comp.bin_file.options.system_libs.keys()[index];
             mingw.buildImportLib(comp, link_lib) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .windows_import_lib,
                     "unable to generate DLL import .lib file: {s}",
                     .{@errorName(err)},
@@ -3076,7 +3109,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             libunwind.buildStaticLib(comp) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .libunwind,
                     "unable to build libunwind: {s}",
                     .{@errorName(err)},
@@ -3089,7 +3122,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             libcxx.buildLibCXX(comp) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .libcxx,
                     "unable to build libcxx: {s}",
                     .{@errorName(err)},
@@ -3102,7 +3135,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             libcxx.buildLibCXXABI(comp) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .libcxxabi,
                     "unable to build libcxxabi: {s}",
                     .{@errorName(err)},
@@ -3115,7 +3148,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             libtsan.buildTsan(comp) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .libtsan,
                     "unable to build TSAN library: {s}",
                     .{@errorName(err)},
@@ -3128,49 +3161,13 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             wasi_libc.buildCRTFile(comp, crt_file) catch |err| {
                 // TODO Surface more error details.
-                try comp.setMiscFailure(
+                comp.lockAndSetMiscFailure(
                     .wasi_libc_crt_file,
                     "unable to build WASI libc CRT file: {s}",
                     .{@errorName(err)},
                 );
             };
         },
-        .compiler_rt_lib => {
-            const named_frame = tracy.namedFrame("compiler_rt_lib");
-            defer named_frame.end();
-
-            compiler_rt.buildCompilerRtLib(
-                comp,
-                &comp.compiler_rt_lib,
-            ) catch |err| switch (err) {
-                error.OutOfMemory => return error.OutOfMemory,
-                error.SubCompilationFailed => return, // error reported already
-                else => try comp.setMiscFailure(
-                    .compiler_rt,
-                    "unable to build compiler_rt: {s}",
-                    .{@errorName(err)},
-                ),
-            };
-        },
-        .compiler_rt_obj => {
-            const named_frame = tracy.namedFrame("compiler_rt_obj");
-            defer named_frame.end();
-
-            comp.buildOutputFromZig(
-                "compiler_rt.zig",
-                .Obj,
-                &comp.compiler_rt_obj,
-                .compiler_rt,
-            ) catch |err| switch (err) {
-                error.OutOfMemory => return error.OutOfMemory,
-                error.SubCompilationFailed => return, // error reported already
-                else => try comp.setMiscFailure(
-                    .compiler_rt,
-                    "unable to build compiler_rt: {s}",
-                    .{@errorName(err)},
-                ),
-            };
-        },
         .libssp => {
             const named_frame = tracy.namedFrame("libssp");
             defer named_frame.end();
@@ -3183,7 +3180,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
             ) catch |err| switch (err) {
                 error.OutOfMemory => return error.OutOfMemory,
                 error.SubCompilationFailed => return, // error reported already
-                else => try comp.setMiscFailure(
+                else => comp.lockAndSetMiscFailure(
                     .libssp,
                     "unable to build libssp: {s}",
                     .{@errorName(err)},
@@ -3202,7 +3199,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
             ) catch |err| switch (err) {
                 error.OutOfMemory => return error.OutOfMemory,
                 error.SubCompilationFailed => return, // error reported already
-                else => try comp.setMiscFailure(
+                else => comp.lockAndSetMiscFailure(
                     .zig_libc,
                     "unable to build zig's multitarget libc: {s}",
                     .{@errorName(err)},
@@ -3306,11 +3303,7 @@ fn workerUpdateBuiltinZigFile(
 
         comp.setMiscFailure(.write_builtin_zig, "unable to write builtin.zig to {s}: {s}", .{
             dir_path, @errorName(err),
-        }) catch |oom| switch (oom) {
-            error.OutOfMemory => log.err("unable to write builtin.zig to {s}: {s}", .{
-                dir_path, @errorName(err),
-            }),
-        };
+        });
     };
 }
 
@@ -3524,6 +3517,38 @@ fn workerUpdateCObject(
     };
 }
 
+fn buildCompilerRtOneShot(
+    comp: *Compilation,
+    output_mode: std.builtin.OutputMode,
+    out: *?CRTFile,
+) void {
+    comp.buildOutputFromZig("compiler_rt.zig", output_mode, out, .compiler_rt) catch |err| switch (err) {
+        error.SubCompilationFailed => return, // error reported already
+        else => comp.lockAndSetMiscFailure(
+            .compiler_rt,
+            "unable to build compiler_rt: {s}",
+            .{@errorName(err)},
+        ),
+    };
+}
+
+fn workerBuildCompilerRtLib(
+    comp: *Compilation,
+    progress_node: *std.Progress.Node,
+    wg: *WaitGroup,
+) void {
+    defer wg.finish();
+
+    compiler_rt.buildCompilerRtLib(comp, progress_node) catch |err| switch (err) {
+        error.SubCompilationFailed => return, // error reported already
+        else => comp.lockAndSetMiscFailure(
+            .compiler_rt,
+            "unable to build compiler_rt: {s}",
+            .{@errorName(err)},
+        ),
+    };
+}
+
 fn reportRetryableCObjectError(
     comp: *Compilation,
     c_object: *CObject,
@@ -4622,14 +4647,21 @@ fn wantBuildLibUnwindFromSource(comp: *Compilation) bool {
         comp.bin_file.options.object_format != .c;
 }
 
-fn setMiscFailure(
+fn setAllocFailure(comp: *Compilation) void {
+    log.debug("memory allocation failure", .{});
+    comp.alloc_failure_occurred = true;
+}
+
+/// Assumes that Compilation mutex is locked.
+/// See also `lockAndSetMiscFailure`.
+pub fn setMiscFailure(
     comp: *Compilation,
     tag: MiscTask,
     comptime format: []const u8,
     args: anytype,
-) Allocator.Error!void {
-    try comp.misc_failures.ensureUnusedCapacity(comp.gpa, 1);
-    const msg = try std.fmt.allocPrint(comp.gpa, format, args);
+) void {
+    comp.misc_failures.ensureUnusedCapacity(comp.gpa, 1) catch return comp.setAllocFailure();
+    const msg = std.fmt.allocPrint(comp.gpa, format, args) catch return comp.setAllocFailure();
     const gop = comp.misc_failures.getOrPutAssumeCapacity(tag);
     if (gop.found_existing) {
         gop.value_ptr.deinit(comp.gpa);
@@ -4637,6 +4669,19 @@ fn setMiscFailure(
     gop.value_ptr.* = .{ .msg = msg };
 }
 
+/// See also `setMiscFailure`.
+pub fn lockAndSetMiscFailure(
+    comp: *Compilation,
+    tag: MiscTask,
+    comptime format: []const u8,
+    args: anytype,
+) void {
+    comp.mutex.lock();
+    defer comp.mutex.unlock();
+
+    return setMiscFailure(comp, tag, format, args);
+}
+
 pub fn dump_argv(argv: []const []const u8) void {
     for (argv[0 .. argv.len - 1]) |arg| {
         std.debug.print("{s} ", .{arg});
@@ -4896,7 +4941,7 @@ pub fn updateSubCompilation(sub_compilation: *Compilation) !void {
     }
 }
 
-pub fn buildOutputFromZig(
+fn buildOutputFromZig(
     comp: *Compilation,
     src_basename: []const u8,
     output_mode: std.builtin.OutputMode,
@@ -4913,15 +4958,7 @@ pub fn buildOutputFromZig(
         .root_src_path = src_basename,
     };
     defer main_pkg.deinitTable(comp.gpa);
-
-    const root_name = root_name: {
-        const basename = if (std.fs.path.dirname(src_basename)) |dirname|
-            src_basename[dirname.len + 1 ..]
-        else
-            src_basename;
-        const root_name = basename[0 .. basename.len - std.fs.path.extension(basename).len];
-        break :root_name root_name;
-    };
+    const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
     const target = comp.getTarget();
     const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
         .root_name = root_name,
src/compiler_rt.zig
@@ -12,316 +12,393 @@ const Compilation = @import("Compilation.zig");
 const CRTFile = Compilation.CRTFile;
 const LinkObject = Compilation.LinkObject;
 const Package = @import("Package.zig");
+const WaitGroup = @import("WaitGroup.zig");
 
-pub fn buildCompilerRtLib(comp: *Compilation, compiler_rt_lib: *?CRTFile) !void {
-    const tracy_trace = trace(@src());
-    defer tracy_trace.end();
-
+pub fn buildCompilerRtLib(comp: *Compilation, progress_node: *std.Progress.Node) !void {
     var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
     defer arena_allocator.deinit();
     const arena = arena_allocator.allocator();
 
     const target = comp.getTarget();
 
-    // Use the global cache directory.
-    var cache_parent: Cache = .{
-        .gpa = comp.gpa,
-        .manifest_dir = try comp.global_cache_directory.handle.makeOpenPath("h", .{}),
+    const root_name = "compiler_rt";
+    const basename = try std.zig.binNameAlloc(arena, .{
+        .root_name = root_name,
+        .target = target,
+        .output_mode = .Lib,
+    });
+
+    var link_objects: [sources.len]LinkObject = undefined;
+    var crt_files = [1]?CRTFile{null} ** sources.len;
+    defer deinitCrtFiles(comp, crt_files);
+
+    {
+        var wg: WaitGroup = .{};
+        defer comp.thread_pool.waitAndWork(&wg);
+
+        for (sources) |source, i| {
+            wg.start();
+            try comp.thread_pool.spawn(workerBuildObject, .{
+                comp, progress_node, &wg, source, &crt_files[i],
+            });
+        }
+    }
+
+    for (link_objects) |*link_object, i| {
+        link_object.* = .{
+            .path = crt_files[i].?.full_object_path,
+        };
+    }
+
+    var link_progress_node = progress_node.start("link", 0);
+    link_progress_node.activate();
+    defer link_progress_node.end();
+
+    // TODO: This is extracted into a local variable to work around a stage1 miscompilation.
+    const emit_bin = Compilation.EmitLoc{
+        .directory = null, // Put it in the cache directory.
+        .basename = basename,
+    };
+    const sub_compilation = try Compilation.create(comp.gpa, .{
+        .local_cache_directory = comp.global_cache_directory,
+        .global_cache_directory = comp.global_cache_directory,
+        .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
+        .target = target,
+        .root_name = root_name,
+        .main_pkg = null,
+        .output_mode = .Lib,
+        .link_mode = .Static,
+        .thread_pool = comp.thread_pool,
+        .libc_installation = comp.bin_file.options.libc_installation,
+        .emit_bin = emit_bin,
+        .optimize_mode = comp.compilerRtOptMode(),
+        .want_sanitize_c = false,
+        .want_stack_check = false,
+        .want_red_zone = comp.bin_file.options.red_zone,
+        .omit_frame_pointer = comp.bin_file.options.omit_frame_pointer,
+        .want_valgrind = false,
+        .want_tsan = false,
+        .want_pic = comp.bin_file.options.pic,
+        .want_pie = comp.bin_file.options.pie,
+        .want_lto = comp.bin_file.options.lto,
+        .emit_h = null,
+        .strip = comp.compilerRtStrip(),
+        .is_native_os = comp.bin_file.options.is_native_os,
+        .is_native_abi = comp.bin_file.options.is_native_abi,
+        .self_exe_path = comp.self_exe_path,
+        .link_objects = &link_objects,
+        .verbose_cc = comp.verbose_cc,
+        .verbose_link = comp.bin_file.options.verbose_link,
+        .verbose_air = comp.verbose_air,
+        .verbose_llvm_ir = comp.verbose_llvm_ir,
+        .verbose_cimport = comp.verbose_cimport,
+        .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
+        .clang_passthrough_mode = comp.clang_passthrough_mode,
+        .skip_linker_dependencies = true,
+        .parent_compilation_link_libc = comp.bin_file.options.link_libc,
+    });
+    defer sub_compilation.destroy();
+
+    try sub_compilation.updateSubCompilation();
+
+    assert(comp.compiler_rt_lib == null);
+    comp.compiler_rt_lib = .{
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
+        }),
+        .lock = sub_compilation.bin_file.toOwnedLock(),
     };
-    defer cache_parent.manifest_dir.close();
+}
 
-    var cache = cache_parent.obtain();
-    defer cache.deinit();
+fn deinitCrtFiles(comp: *Compilation, crt_files: [sources.len]?CRTFile) void {
+    const gpa = comp.gpa;
 
-    cache.hash.add(sources.len);
-    for (sources) |source| {
-        const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{source});
-        _ = try cache.addFile(full_path, null);
+    for (crt_files) |opt_crt_file| {
+        var crt_file = opt_crt_file orelse continue;
+        crt_file.deinit(gpa);
     }
+}
 
-    cache.hash.addBytes(build_options.version);
-    cache.hash.addBytes(comp.zig_lib_directory.path orelse ".");
-    cache.hash.add(target.cpu.arch);
-    cache.hash.add(target.os.tag);
-    cache.hash.add(target.abi);
+fn workerBuildObject(
+    comp: *Compilation,
+    progress_node: *std.Progress.Node,
+    wg: *WaitGroup,
+    src_basename: []const u8,
+    out: *?CRTFile,
+) void {
+    defer wg.finish();
 
-    const hit = try cache.hit();
-    const digest = cache.final();
-    const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
+    var obj_progress_node = progress_node.start(src_basename, 0);
+    obj_progress_node.activate();
+    defer obj_progress_node.end();
 
-    var o_directory: Compilation.Directory = .{
-        .handle = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{}),
-        .path = try std.fs.path.join(arena, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }),
+    buildObject(comp, src_basename, out) catch |err| switch (err) {
+        error.SubCompilationFailed => return, // error reported already
+        else => comp.lockAndSetMiscFailure(
+            .compiler_rt,
+            "unable to build compiler_rt: {s}",
+            .{@errorName(err)},
+        ),
     };
-    defer o_directory.handle.close();
+}
 
-    const ok_basename = "ok";
-    const actual_hit = if (hit) blk: {
-        o_directory.handle.access(ok_basename, .{}) catch |err| switch (err) {
-            error.FileNotFound => break :blk false,
-            else => |e| return e,
-        };
-        break :blk true;
-    } else false;
+fn buildObject(comp: *Compilation, src_basename: []const u8, out: *?CRTFile) !void {
+    const gpa = comp.gpa;
 
-    const root_name = "compiler_rt";
-    const basename = try std.zig.binNameAlloc(arena, .{
+    var root_src_path_buf: [64]u8 = undefined;
+    const root_src_path = std.fmt.bufPrint(
+        &root_src_path_buf,
+        "compiler_rt" ++ std.fs.path.sep_str ++ "{s}",
+        .{src_basename},
+    ) catch unreachable;
+
+    var main_pkg: Package = .{
+        .root_src_directory = comp.zig_lib_directory,
+        .root_src_path = root_src_path,
+    };
+    defer main_pkg.deinitTable(gpa);
+    const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
+    const target = comp.getTarget();
+    const output_mode: std.builtin.OutputMode = .Obj;
+    const bin_basename = try std.zig.binNameAlloc(gpa, .{
         .root_name = root_name,
         .target = target,
-        .output_mode = .Lib,
+        .output_mode = output_mode,
     });
+    defer gpa.free(bin_basename);
 
-    if (!actual_hit) {
-        var progress: std.Progress = .{ .dont_print_on_dumb = true };
-        var progress_node = progress.start("Compile Compiler-RT", sources.len + 1);
-        defer progress_node.end();
-        if (comp.color == .off) progress.terminal = null;
-
-        progress_node.activate();
+    const emit_bin = Compilation.EmitLoc{
+        .directory = null, // Put it in the cache directory.
+        .basename = bin_basename,
+    };
+    const sub_compilation = try Compilation.create(gpa, .{
+        .global_cache_directory = comp.global_cache_directory,
+        .local_cache_directory = comp.global_cache_directory,
+        .zig_lib_directory = comp.zig_lib_directory,
+        .cache_mode = .whole,
+        .target = target,
+        .root_name = root_name,
+        .main_pkg = &main_pkg,
+        .output_mode = output_mode,
+        .thread_pool = comp.thread_pool,
+        .libc_installation = comp.bin_file.options.libc_installation,
+        .emit_bin = emit_bin,
+        .optimize_mode = comp.compilerRtOptMode(),
+        .link_mode = .Static,
+        .want_sanitize_c = false,
+        .want_stack_check = false,
+        .want_red_zone = comp.bin_file.options.red_zone,
+        .omit_frame_pointer = comp.bin_file.options.omit_frame_pointer,
+        .want_valgrind = false,
+        .want_tsan = false,
+        .want_pic = comp.bin_file.options.pic,
+        .want_pie = comp.bin_file.options.pie,
+        .emit_h = null,
+        .strip = comp.compilerRtStrip(),
+        .is_native_os = comp.bin_file.options.is_native_os,
+        .is_native_abi = comp.bin_file.options.is_native_abi,
+        .self_exe_path = comp.self_exe_path,
+        .verbose_cc = comp.verbose_cc,
+        .verbose_link = comp.bin_file.options.verbose_link,
+        .verbose_air = comp.verbose_air,
+        .verbose_llvm_ir = comp.verbose_llvm_ir,
+        .verbose_cimport = comp.verbose_cimport,
+        .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
+        .clang_passthrough_mode = comp.clang_passthrough_mode,
+        .skip_linker_dependencies = true,
+        .parent_compilation_link_libc = comp.bin_file.options.link_libc,
+    });
+    defer sub_compilation.destroy();
 
-        var link_objects: [sources.len]LinkObject = undefined;
-        for (sources) |source, i| {
-            var obj_progress_node = progress_node.start(source, 0);
-            obj_progress_node.activate();
-            defer obj_progress_node.end();
+    try sub_compilation.update();
+    // Look for compilation errors in this sub_compilation.
+    var keep_errors = false;
+    var errors = try sub_compilation.getAllErrorsAlloc();
+    defer if (!keep_errors) errors.deinit(sub_compilation.gpa);
 
-            var tmp_crt_file: ?CRTFile = null;
-            defer if (tmp_crt_file) |*crt| crt.deinit(comp.gpa);
-            try comp.buildOutputFromZig(source, .Obj, &tmp_crt_file, .compiler_rt);
-            link_objects[i] = .{
-                .path = try arena.dupe(u8, tmp_crt_file.?.full_object_path),
-                .must_link = true,
-            };
-        }
+    if (errors.list.len != 0) {
+        const misc_task_tag: Compilation.MiscTask = .compiler_rt;
 
-        var lib_progress_node = progress_node.start(root_name, 0);
-        lib_progress_node.activate();
-        defer lib_progress_node.end();
+        comp.mutex.lock();
+        defer comp.mutex.unlock();
 
-        // TODO: This is extracted into a local variable to work around a stage1 miscompilation.
-        const emit_bin = Compilation.EmitLoc{
-            .directory = o_directory, // Put it in the cache directory.
-            .basename = basename,
-        };
-        const sub_compilation = try Compilation.create(comp.gpa, .{
-            .local_cache_directory = comp.global_cache_directory,
-            .global_cache_directory = comp.global_cache_directory,
-            .zig_lib_directory = comp.zig_lib_directory,
-            .cache_mode = .whole,
-            .target = target,
-            .root_name = root_name,
-            .main_pkg = null,
-            .output_mode = .Lib,
-            .link_mode = .Static,
-            .thread_pool = comp.thread_pool,
-            .libc_installation = comp.bin_file.options.libc_installation,
-            .emit_bin = emit_bin,
-            .optimize_mode = comp.compilerRtOptMode(),
-            .want_sanitize_c = false,
-            .want_stack_check = false,
-            .want_red_zone = comp.bin_file.options.red_zone,
-            .omit_frame_pointer = comp.bin_file.options.omit_frame_pointer,
-            .want_valgrind = false,
-            .want_tsan = false,
-            .want_pic = comp.bin_file.options.pic,
-            .want_pie = comp.bin_file.options.pie,
-            .want_lto = comp.bin_file.options.lto,
-            .emit_h = null,
-            .strip = comp.compilerRtStrip(),
-            .is_native_os = comp.bin_file.options.is_native_os,
-            .is_native_abi = comp.bin_file.options.is_native_abi,
-            .self_exe_path = comp.self_exe_path,
-            .link_objects = &link_objects,
-            .verbose_cc = comp.verbose_cc,
-            .verbose_link = comp.bin_file.options.verbose_link,
-            .verbose_air = comp.verbose_air,
-            .verbose_llvm_ir = comp.verbose_llvm_ir,
-            .verbose_cimport = comp.verbose_cimport,
-            .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
-            .clang_passthrough_mode = comp.clang_passthrough_mode,
-            .skip_linker_dependencies = true,
-            .parent_compilation_link_libc = comp.bin_file.options.link_libc,
+        try comp.misc_failures.ensureUnusedCapacity(gpa, 1);
+        comp.misc_failures.putAssumeCapacityNoClobber(misc_task_tag, .{
+            .msg = try std.fmt.allocPrint(gpa, "sub-compilation of {s} failed", .{
+                @tagName(misc_task_tag),
+            }),
+            .children = errors,
         });
-        defer sub_compilation.destroy();
-
-        try sub_compilation.updateSubCompilation();
-
-        if (o_directory.handle.createFile(ok_basename, .{})) |file| {
-            file.close();
-        } else |err| {
-            std.log.warn("compiler-rt lib: failed to mark completion: {s}", .{@errorName(err)});
-        }
+        keep_errors = true;
+        return error.SubCompilationFailed;
     }
 
-    try cache.writeManifest();
-
-    assert(compiler_rt_lib.* == null);
-    compiler_rt_lib.* = .{
-        .full_object_path = try std.fs.path.join(comp.gpa, &[_][]const u8{
-            comp.global_cache_directory.path.?,
-            o_sub_path,
-            basename,
+    assert(out.* == null);
+    out.* = Compilation.CRTFile{
+        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(gpa, &[_][]const u8{
+            sub_compilation.bin_file.options.emit.?.sub_path,
         }),
-        .lock = cache.toOwnedLock(),
+        .lock = sub_compilation.bin_file.toOwnedLock(),
     };
 }
 
-const sources = &[_][]const u8{
-    "compiler_rt/absvdi2.zig",
-    "compiler_rt/absvsi2.zig",
-    "compiler_rt/absvti2.zig",
-    "compiler_rt/adddf3.zig",
-    "compiler_rt/addo.zig",
-    "compiler_rt/addsf3.zig",
-    "compiler_rt/addtf3.zig",
-    "compiler_rt/addxf3.zig",
-    "compiler_rt/arm.zig",
-    "compiler_rt/atomics.zig",
-    "compiler_rt/aulldiv.zig",
-    "compiler_rt/aullrem.zig",
-    "compiler_rt/bswap.zig",
-    "compiler_rt/ceil.zig",
-    "compiler_rt/clear_cache.zig",
-    "compiler_rt/cmp.zig",
-    "compiler_rt/cmpdf2.zig",
-    "compiler_rt/cmpsf2.zig",
-    "compiler_rt/cmptf2.zig",
-    "compiler_rt/cmpxf2.zig",
-    "compiler_rt/cos.zig",
-    "compiler_rt/count0bits.zig",
-    "compiler_rt/divdf3.zig",
-    "compiler_rt/divsf3.zig",
-    "compiler_rt/divtf3.zig",
-    "compiler_rt/divti3.zig",
-    "compiler_rt/divxf3.zig",
-    "compiler_rt/emutls.zig",
-    "compiler_rt/exp.zig",
-    "compiler_rt/exp2.zig",
-    "compiler_rt/extenddftf2.zig",
-    "compiler_rt/extenddfxf2.zig",
-    "compiler_rt/extendhfsf2.zig",
-    "compiler_rt/extendhftf2.zig",
-    "compiler_rt/extendhfxf2.zig",
-    "compiler_rt/extendsfdf2.zig",
-    "compiler_rt/extendsftf2.zig",
-    "compiler_rt/extendsfxf2.zig",
-    "compiler_rt/extendxftf2.zig",
-    "compiler_rt/fabs.zig",
-    "compiler_rt/fixdfdi.zig",
-    "compiler_rt/fixdfsi.zig",
-    "compiler_rt/fixdfti.zig",
-    "compiler_rt/fixhfdi.zig",
-    "compiler_rt/fixhfsi.zig",
-    "compiler_rt/fixhfti.zig",
-    "compiler_rt/fixsfdi.zig",
-    "compiler_rt/fixsfsi.zig",
-    "compiler_rt/fixsfti.zig",
-    "compiler_rt/fixtfdi.zig",
-    "compiler_rt/fixtfsi.zig",
-    "compiler_rt/fixtfti.zig",
-    "compiler_rt/fixunsdfdi.zig",
-    "compiler_rt/fixunsdfsi.zig",
-    "compiler_rt/fixunsdfti.zig",
-    "compiler_rt/fixunshfdi.zig",
-    "compiler_rt/fixunshfsi.zig",
-    "compiler_rt/fixunshfti.zig",
-    "compiler_rt/fixunssfdi.zig",
-    "compiler_rt/fixunssfsi.zig",
-    "compiler_rt/fixunssfti.zig",
-    "compiler_rt/fixunstfdi.zig",
-    "compiler_rt/fixunstfsi.zig",
-    "compiler_rt/fixunstfti.zig",
-    "compiler_rt/fixunsxfdi.zig",
-    "compiler_rt/fixunsxfsi.zig",
-    "compiler_rt/fixunsxfti.zig",
-    "compiler_rt/fixxfdi.zig",
-    "compiler_rt/fixxfsi.zig",
-    "compiler_rt/fixxfti.zig",
-    "compiler_rt/floatdidf.zig",
-    "compiler_rt/floatdihf.zig",
-    "compiler_rt/floatdisf.zig",
-    "compiler_rt/floatditf.zig",
-    "compiler_rt/floatdixf.zig",
-    "compiler_rt/floatsidf.zig",
-    "compiler_rt/floatsihf.zig",
-    "compiler_rt/floatsisf.zig",
-    "compiler_rt/floatsitf.zig",
-    "compiler_rt/floatsixf.zig",
-    "compiler_rt/floattidf.zig",
-    "compiler_rt/floattihf.zig",
-    "compiler_rt/floattisf.zig",
-    "compiler_rt/floattitf.zig",
-    "compiler_rt/floattixf.zig",
-    "compiler_rt/floatundidf.zig",
-    "compiler_rt/floatundihf.zig",
-    "compiler_rt/floatundisf.zig",
-    "compiler_rt/floatunditf.zig",
-    "compiler_rt/floatundixf.zig",
-    "compiler_rt/floatunsidf.zig",
-    "compiler_rt/floatunsihf.zig",
-    "compiler_rt/floatunsisf.zig",
-    "compiler_rt/floatunsitf.zig",
-    "compiler_rt/floatunsixf.zig",
-    "compiler_rt/floatuntidf.zig",
-    "compiler_rt/floatuntihf.zig",
-    "compiler_rt/floatuntisf.zig",
-    "compiler_rt/floatuntitf.zig",
-    "compiler_rt/floatuntixf.zig",
-    "compiler_rt/floor.zig",
-    "compiler_rt/fma.zig",
-    "compiler_rt/fmax.zig",
-    "compiler_rt/fmin.zig",
-    "compiler_rt/fmod.zig",
-    "compiler_rt/gedf2.zig",
-    "compiler_rt/gesf2.zig",
-    "compiler_rt/getf2.zig",
-    "compiler_rt/gexf2.zig",
-    "compiler_rt/int.zig",
-    "compiler_rt/log.zig",
-    "compiler_rt/log10.zig",
-    "compiler_rt/log2.zig",
-    "compiler_rt/modti3.zig",
-    "compiler_rt/muldf3.zig",
-    "compiler_rt/muldi3.zig",
-    "compiler_rt/mulf3.zig",
-    "compiler_rt/mulo.zig",
-    "compiler_rt/mulsf3.zig",
-    "compiler_rt/multf3.zig",
-    "compiler_rt/multi3.zig",
-    "compiler_rt/mulxf3.zig",
-    "compiler_rt/negXf2.zig",
-    "compiler_rt/negXi2.zig",
-    "compiler_rt/negv.zig",
-    "compiler_rt/os_version_check.zig",
-    "compiler_rt/parity.zig",
-    "compiler_rt/popcount.zig",
-    "compiler_rt/round.zig",
-    "compiler_rt/shift.zig",
-    "compiler_rt/sin.zig",
-    "compiler_rt/sincos.zig",
-    "compiler_rt/sqrt.zig",
-    "compiler_rt/stack_probe.zig",
-    "compiler_rt/subdf3.zig",
-    "compiler_rt/subo.zig",
-    "compiler_rt/subsf3.zig",
-    "compiler_rt/subtf3.zig",
-    "compiler_rt/subxf3.zig",
-    "compiler_rt/tan.zig",
-    "compiler_rt/trunc.zig",
-    "compiler_rt/truncdfhf2.zig",
-    "compiler_rt/truncdfsf2.zig",
-    "compiler_rt/truncsfhf2.zig",
-    "compiler_rt/trunctfdf2.zig",
-    "compiler_rt/trunctfhf2.zig",
-    "compiler_rt/trunctfsf2.zig",
-    "compiler_rt/trunctfxf2.zig",
-    "compiler_rt/truncxfdf2.zig",
-    "compiler_rt/truncxfhf2.zig",
-    "compiler_rt/truncxfsf2.zig",
-    "compiler_rt/udivmodti4.zig",
-    "compiler_rt/udivti3.zig",
-    "compiler_rt/umodti3.zig",
-    "compiler_rt/unorddf2.zig",
-    "compiler_rt/unordsf2.zig",
-    "compiler_rt/unordtf2.zig",
+pub const sources = &[_][]const u8{
+    "absvdi2.zig",
+    "absvsi2.zig",
+    "absvti2.zig",
+    "adddf3.zig",
+    "addo.zig",
+    "addsf3.zig",
+    "addtf3.zig",
+    "addxf3.zig",
+    "arm.zig",
+    "atomics.zig",
+    "aulldiv.zig",
+    "aullrem.zig",
+    "bswap.zig",
+    "ceil.zig",
+    "clear_cache.zig",
+    "cmp.zig",
+    "cmpdf2.zig",
+    "cmpsf2.zig",
+    "cmptf2.zig",
+    "cmpxf2.zig",
+    "cos.zig",
+    "count0bits.zig",
+    "divdf3.zig",
+    "divsf3.zig",
+    "divtf3.zig",
+    "divti3.zig",
+    "divxf3.zig",
+    "emutls.zig",
+    "exp.zig",
+    "exp2.zig",
+    "extenddftf2.zig",
+    "extenddfxf2.zig",
+    "extendhfsf2.zig",
+    "extendhftf2.zig",
+    "extendhfxf2.zig",
+    "extendsfdf2.zig",
+    "extendsftf2.zig",
+    "extendsfxf2.zig",
+    "extendxftf2.zig",
+    "fabs.zig",
+    "fixdfdi.zig",
+    "fixdfsi.zig",
+    "fixdfti.zig",
+    "fixhfdi.zig",
+    "fixhfsi.zig",
+    "fixhfti.zig",
+    "fixsfdi.zig",
+    "fixsfsi.zig",
+    "fixsfti.zig",
+    "fixtfdi.zig",
+    "fixtfsi.zig",
+    "fixtfti.zig",
+    "fixunsdfdi.zig",
+    "fixunsdfsi.zig",
+    "fixunsdfti.zig",
+    "fixunshfdi.zig",
+    "fixunshfsi.zig",
+    "fixunshfti.zig",
+    "fixunssfdi.zig",
+    "fixunssfsi.zig",
+    "fixunssfti.zig",
+    "fixunstfdi.zig",
+    "fixunstfsi.zig",
+    "fixunstfti.zig",
+    "fixunsxfdi.zig",
+    "fixunsxfsi.zig",
+    "fixunsxfti.zig",
+    "fixxfdi.zig",
+    "fixxfsi.zig",
+    "fixxfti.zig",
+    "floatdidf.zig",
+    "floatdihf.zig",
+    "floatdisf.zig",
+    "floatditf.zig",
+    "floatdixf.zig",
+    "floatsidf.zig",
+    "floatsihf.zig",
+    "floatsisf.zig",
+    "floatsitf.zig",
+    "floatsixf.zig",
+    "floattidf.zig",
+    "floattihf.zig",
+    "floattisf.zig",
+    "floattitf.zig",
+    "floattixf.zig",
+    "floatundidf.zig",
+    "floatundihf.zig",
+    "floatundisf.zig",
+    "floatunditf.zig",
+    "floatundixf.zig",
+    "floatunsidf.zig",
+    "floatunsihf.zig",
+    "floatunsisf.zig",
+    "floatunsitf.zig",
+    "floatunsixf.zig",
+    "floatuntidf.zig",
+    "floatuntihf.zig",
+    "floatuntisf.zig",
+    "floatuntitf.zig",
+    "floatuntixf.zig",
+    "floor.zig",
+    "fma.zig",
+    "fmax.zig",
+    "fmin.zig",
+    "fmod.zig",
+    "gedf2.zig",
+    "gesf2.zig",
+    "getf2.zig",
+    "gexf2.zig",
+    "int.zig",
+    "log.zig",
+    "log10.zig",
+    "log2.zig",
+    "modti3.zig",
+    "muldf3.zig",
+    "muldi3.zig",
+    "mulf3.zig",
+    "mulo.zig",
+    "mulsf3.zig",
+    "multf3.zig",
+    "multi3.zig",
+    "mulxf3.zig",
+    "negXf2.zig",
+    "negXi2.zig",
+    "negv.zig",
+    "os_version_check.zig",
+    "parity.zig",
+    "popcount.zig",
+    "round.zig",
+    "shift.zig",
+    "sin.zig",
+    "sincos.zig",
+    "sqrt.zig",
+    "stack_probe.zig",
+    "subdf3.zig",
+    "subo.zig",
+    "subsf3.zig",
+    "subtf3.zig",
+    "subxf3.zig",
+    "tan.zig",
+    "trunc.zig",
+    "truncdfhf2.zig",
+    "truncdfsf2.zig",
+    "truncsfhf2.zig",
+    "trunctfdf2.zig",
+    "trunctfhf2.zig",
+    "trunctfsf2.zig",
+    "trunctfxf2.zig",
+    "truncxfdf2.zig",
+    "truncxfhf2.zig",
+    "truncxfsf2.zig",
+    "udivmodti4.zig",
+    "udivti3.zig",
+    "umodti3.zig",
+    "unorddf2.zig",
+    "unordsf2.zig",
+    "unordtf2.zig",
 };
src/ThreadPool.zig
@@ -1,6 +1,7 @@
 const std = @import("std");
 const builtin = @import("builtin");
 const ThreadPool = @This();
+const WaitGroup = @import("WaitGroup.zig");
 
 mutex: std.Thread.Mutex = .{},
 cond: std.Thread.Condition = .{},
@@ -19,8 +20,8 @@ const RunProto = switch (builtin.zig_backend) {
     else => *const fn (*Runnable) void,
 };
 
-pub fn init(self: *ThreadPool, allocator: std.mem.Allocator) !void {
-    self.* = .{
+pub fn init(pool: *ThreadPool, allocator: std.mem.Allocator) !void {
+    pool.* = .{
         .allocator = allocator,
         .threads = &[_]std.Thread{},
     };
@@ -30,48 +31,48 @@ pub fn init(self: *ThreadPool, allocator: std.mem.Allocator) !void {
     }
 
     const thread_count = std.math.max(1, std.Thread.getCpuCount() catch 1);
-    self.threads = try allocator.alloc(std.Thread, thread_count);
-    errdefer allocator.free(self.threads);
+    pool.threads = try allocator.alloc(std.Thread, thread_count);
+    errdefer allocator.free(pool.threads);
 
     // kill and join any threads we spawned previously on error.
     var spawned: usize = 0;
-    errdefer self.join(spawned);
+    errdefer pool.join(spawned);
 
-    for (self.threads) |*thread| {
-        thread.* = try std.Thread.spawn(.{}, worker, .{self});
+    for (pool.threads) |*thread| {
+        thread.* = try std.Thread.spawn(.{}, worker, .{pool});
         spawned += 1;
     }
 }
 
-pub fn deinit(self: *ThreadPool) void {
-    self.join(self.threads.len); // kill and join all threads.
-    self.* = undefined;
+pub fn deinit(pool: *ThreadPool) void {
+    pool.join(pool.threads.len); // kill and join all threads.
+    pool.* = undefined;
 }
 
-fn join(self: *ThreadPool, spawned: usize) void {
+fn join(pool: *ThreadPool, spawned: usize) void {
     if (builtin.single_threaded) {
         return;
     }
 
     {
-        self.mutex.lock();
-        defer self.mutex.unlock();
+        pool.mutex.lock();
+        defer pool.mutex.unlock();
 
         // ensure future worker threads exit the dequeue loop
-        self.is_running = false;
+        pool.is_running = false;
     }
 
     // wake up any sleeping threads (this can be done outside the mutex)
     // then wait for all the threads we know are spawned to complete.
-    self.cond.broadcast();
-    for (self.threads[0..spawned]) |thread| {
+    pool.cond.broadcast();
+    for (pool.threads[0..spawned]) |thread| {
         thread.join();
     }
 
-    self.allocator.free(self.threads);
+    pool.allocator.free(pool.threads);
 }
 
-pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
+pub fn spawn(pool: *ThreadPool, comptime func: anytype, args: anytype) !void {
     if (builtin.single_threaded) {
         @call(.{}, func, args);
         return;
@@ -98,41 +99,57 @@ pub fn spawn(self: *ThreadPool, comptime func: anytype, args: anytype) !void {
     };
 
     {
-        self.mutex.lock();
-        defer self.mutex.unlock();
+        pool.mutex.lock();
+        defer pool.mutex.unlock();
 
-        const closure = try self.allocator.create(Closure);
+        const closure = try pool.allocator.create(Closure);
         closure.* = .{
             .arguments = args,
-            .pool = self,
+            .pool = pool,
         };
 
-        self.run_queue.prepend(&closure.run_node);
+        pool.run_queue.prepend(&closure.run_node);
     }
 
     // Notify waiting threads outside the lock to try and keep the critical section small.
-    self.cond.signal();
+    pool.cond.signal();
 }
 
-fn worker(self: *ThreadPool) void {
-    self.mutex.lock();
-    defer self.mutex.unlock();
+fn worker(pool: *ThreadPool) void {
+    pool.mutex.lock();
+    defer pool.mutex.unlock();
 
     while (true) {
-        while (self.run_queue.popFirst()) |run_node| {
+        while (pool.run_queue.popFirst()) |run_node| {
             // Temporarily unlock the mutex in order to execute the run_node
-            self.mutex.unlock();
-            defer self.mutex.lock();
+            pool.mutex.unlock();
+            defer pool.mutex.lock();
 
             const runFn = run_node.data.runFn;
             runFn(&run_node.data);
         }
 
         // Stop executing instead of waiting if the thread pool is no longer running.
-        if (self.is_running) {
-            self.cond.wait(&self.mutex);
+        if (pool.is_running) {
+            pool.cond.wait(&pool.mutex);
         } else {
             break;
         }
     }
 }
+
+pub fn waitAndWork(pool: *ThreadPool, wait_group: *WaitGroup) void {
+    while (!wait_group.isDone()) {
+        if (blk: {
+            pool.mutex.lock();
+            defer pool.mutex.unlock();
+            break :blk pool.run_queue.popFirst();
+        }) |run_node| {
+            run_node.data.runFn(&run_node.data);
+            continue;
+        }
+
+        wait_group.wait();
+        return;
+    }
+}
src/WaitGroup.zig
@@ -37,3 +37,10 @@ pub fn reset(self: *WaitGroup) void {
     self.state.store(0, .Monotonic);
     self.event.reset();
 }
+
+pub fn isDone(wg: *WaitGroup) bool {
+    const state = wg.state.load(.Acquire);
+    assert(state & is_waiting == 0);
+
+    return (state / one_pending) == 0;
+}