Commit 43f73af359
Changed files (16)
lib
compiler
tools
lib/compiler/objcopy.zig
@@ -201,9 +201,10 @@ fn cmdObjCopy(
if (seen_update) fatal("zig objcopy only supports 1 update for now", .{});
seen_update = true;
- try server.serveEmitBinPath(output, .{
- .flags = .{ .cache_hit = false },
- });
+ // The build system already knows what the output is at this point, we
+ // only need to communicate that the process has finished.
+ // Use the empty error bundle to indicate that the update is done.
+ try server.serveErrorBundle(std.zig.ErrorBundle.empty);
},
else => fatal("unsupported message: {s}", .{@tagName(hdr.tag)}),
}
lib/std/Build/Fuzz/WebServer.zig
@@ -8,6 +8,8 @@ const Coverage = std.debug.Coverage;
const abi = std.Build.Fuzz.abi;
const log = std.log;
const assert = std.debug.assert;
+const Cache = std.Build.Cache;
+const Path = Cache.Path;
const WebServer = @This();
@@ -31,6 +33,10 @@ coverage_mutex: std.Thread.Mutex,
/// Signaled when `coverage_files` changes.
coverage_condition: std.Thread.Condition,
+const fuzzer_bin_name = "fuzzer";
+const fuzzer_arch_os_abi = "wasm32-freestanding";
+const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
+
const CoverageMap = struct {
mapped_memory: []align(std.mem.page_size) const u8,
coverage: Coverage,
@@ -181,9 +187,18 @@ fn serveWasm(
// Do the compilation every request, so that the user can edit the files
// and see the changes without restarting the server.
- const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);
+ const wasm_base_path = try buildWasmBinary(ws, arena, optimize_mode);
+ const bin_name = try std.zig.binNameAlloc(arena, .{
+ .root_name = fuzzer_bin_name,
+ .target = std.zig.system.resolveTargetQuery(std.Build.parseTargetQuery(.{
+ .arch_os_abi = fuzzer_arch_os_abi,
+ .cpu_features = fuzzer_cpu_features,
+ }) catch unreachable) catch unreachable,
+ .output_mode = .Exe,
+ });
// std.http.Server does not have a sendfile API yet.
- const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);
+ const bin_path = try wasm_base_path.join(arena, bin_name);
+ const file_contents = try bin_path.root_dir.handle.readFileAlloc(gpa, bin_path.sub_path, 10 * 1024 * 1024);
defer gpa.free(file_contents);
try request.respond(file_contents, .{
.extra_headers = &.{
@@ -197,7 +212,7 @@ fn buildWasmBinary(
ws: *WebServer,
arena: Allocator,
optimize_mode: std.builtin.OptimizeMode,
-) ![]const u8 {
+) !Path {
const gpa = ws.gpa;
const main_src_path: Build.Cache.Path = .{
@@ -219,11 +234,11 @@ fn buildWasmBinary(
ws.zig_exe_path, "build-exe", //
"-fno-entry", //
"-O", @tagName(optimize_mode), //
- "-target", "wasm32-freestanding", //
- "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
+ "-target", fuzzer_arch_os_abi, //
+ "-mcpu", fuzzer_cpu_features, //
"--cache-dir", ws.global_cache_directory.path orelse ".", //
"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
- "--name", "fuzzer", //
+ "--name", fuzzer_bin_name, //
"-rdynamic", //
"-fsingle-threaded", //
"--dep", "Walk", //
@@ -251,7 +266,7 @@ fn buildWasmBinary(
try sendMessage(child.stdin.?, .exit);
const Header = std.zig.Server.Message.Header;
- var result: ?[]const u8 = null;
+ var result: ?Path = null;
var result_error_bundle = std.zig.ErrorBundle.empty;
const stdout = poller.fifo(.stdout);
@@ -288,13 +303,17 @@ fn buildWasmBinary(
.extra = extra_array,
};
},
- .emit_bin_path => {
- const EbpHdr = std.zig.Server.Message.EmitBinPath;
+ .emit_digest => {
+ const EbpHdr = std.zig.Server.Message.EmitDigest;
const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
if (!ebp_hdr.flags.cache_hit) {
log.info("source changes detected; rebuilt wasm component", .{});
}
- result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+ const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
+ result = Path{
+ .root_dir = ws.global_cache_directory,
+ .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
+ };
},
else => {}, // ignore other messages
}
@@ -568,10 +587,7 @@ fn prepareTables(
};
errdefer gop.value_ptr.coverage.deinit(gpa);
- const rebuilt_exe_path: Build.Cache.Path = .{
- .root_dir = Build.Cache.Directory.cwd(),
- .sub_path = run_step.rebuilt_executable.?,
- };
+ const rebuilt_exe_path = run_step.rebuilt_executable.?;
var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
log.err("step '{s}': failed to load debug information for '{}': {s}", .{
run_step.step.name, rebuilt_exe_path, @errorName(err),
lib/std/Build/Step/Compile.zig
@@ -17,6 +17,7 @@ const Module = std.Build.Module;
const InstallDir = std.Build.InstallDir;
const GeneratedFile = std.Build.GeneratedFile;
const Compile = @This();
+const Path = std.Build.Cache.Path;
pub const base_id: Step.Id = .compile;
@@ -1765,7 +1766,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
const zig_args = try getZigArgs(compile, false);
- const maybe_output_bin_path = step.evalZigProcess(
+ const maybe_output_dir = step.evalZigProcess(
zig_args,
options.progress_node,
(b.graph.incremental == true) and options.watch,
@@ -1779,53 +1780,51 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
};
// Update generated files
- if (maybe_output_bin_path) |output_bin_path| {
- const output_dir = fs.path.dirname(output_bin_path).?;
-
+ if (maybe_output_dir) |output_dir| {
if (compile.emit_directory) |lp| {
- lp.path = output_dir;
+ lp.path = b.fmt("{}", .{output_dir});
}
// -femit-bin[=path] (default) Output machine code
if (compile.generated_bin) |bin| {
- bin.path = b.pathJoin(&.{ output_dir, compile.out_filename });
+ bin.path = output_dir.joinString(b.allocator, compile.out_filename) catch @panic("OOM");
}
- const sep = std.fs.path.sep;
+ const sep = std.fs.path.sep_str;
// output PDB if someone requested it
if (compile.generated_pdb) |pdb| {
- pdb.path = b.fmt("{s}{c}{s}.pdb", .{ output_dir, sep, compile.name });
+ pdb.path = b.fmt("{}" ++ sep ++ "{s}.pdb", .{ output_dir, compile.name });
}
// -femit-implib[=path] (default) Produce an import .lib when building a Windows DLL
if (compile.generated_implib) |implib| {
- implib.path = b.fmt("{s}{c}{s}.lib", .{ output_dir, sep, compile.name });
+ implib.path = b.fmt("{}" ++ sep ++ "{s}.lib", .{ output_dir, compile.name });
}
// -femit-h[=path] Generate a C header file (.h)
if (compile.generated_h) |lp| {
- lp.path = b.fmt("{s}{c}{s}.h", .{ output_dir, sep, compile.name });
+ lp.path = b.fmt("{}" ++ sep ++ "{s}.h", .{ output_dir, compile.name });
}
// -femit-docs[=path] Create a docs/ dir with html documentation
if (compile.generated_docs) |generated_docs| {
- generated_docs.path = b.pathJoin(&.{ output_dir, "docs" });
+ generated_docs.path = output_dir.joinString(b.allocator, "docs") catch @panic("OOM");
}
// -femit-asm[=path] Output .s (assembly code)
if (compile.generated_asm) |lp| {
- lp.path = b.fmt("{s}{c}{s}.s", .{ output_dir, sep, compile.name });
+ lp.path = b.fmt("{}" ++ sep ++ "{s}.s", .{ output_dir, compile.name });
}
// -femit-llvm-ir[=path] Produce a .ll file with optimized LLVM IR (requires LLVM extensions)
if (compile.generated_llvm_ir) |lp| {
- lp.path = b.fmt("{s}{c}{s}.ll", .{ output_dir, sep, compile.name });
+ lp.path = b.fmt("{}" ++ sep ++ "{s}.ll", .{ output_dir, compile.name });
}
// -femit-llvm-bc[=path] Produce an optimized LLVM module as a .bc file (requires LLVM extensions)
if (compile.generated_llvm_bc) |lp| {
- lp.path = b.fmt("{s}{c}{s}.bc", .{ output_dir, sep, compile.name });
+ lp.path = b.fmt("{}" ++ sep ++ "{s}.bc", .{ output_dir, compile.name });
}
}
@@ -1841,7 +1840,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
}
}
-pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
+pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path {
const gpa = c.step.owner.allocator;
c.step.result_error_msgs.clearRetainingCapacity();
lib/std/Build/Step/InstallArtifact.zig
@@ -125,10 +125,10 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
if (install_artifact.dest_dir) |dest_dir| {
const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path);
- const full_src_path = install_artifact.emitted_bin.?.getPath2(b, step);
- const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_dest_path, .{}) catch |err| {
+ const src_path = install_artifact.emitted_bin.?.getPath3(b, step);
+ const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_dest_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_path, full_dest_path, @errorName(err),
+ src_path.sub_path, full_dest_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
@@ -141,22 +141,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
}
if (install_artifact.implib_dir) |implib_dir| {
- const full_src_path = install_artifact.emitted_implib.?.getPath2(b, step);
- const full_implib_path = b.getInstallPath(implib_dir, fs.path.basename(full_src_path));
- const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_implib_path, .{}) catch |err| {
+ const src_path = install_artifact.emitted_implib.?.getPath3(b, step);
+ const full_implib_path = b.getInstallPath(implib_dir, fs.path.basename(src_path.sub_path));
+ const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_implib_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_path, full_implib_path, @errorName(err),
+ src_path.sub_path, full_implib_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
}
if (install_artifact.pdb_dir) |pdb_dir| {
- const full_src_path = install_artifact.emitted_pdb.?.getPath2(b, step);
- const full_pdb_path = b.getInstallPath(pdb_dir, fs.path.basename(full_src_path));
- const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_pdb_path, .{}) catch |err| {
+ const src_path = install_artifact.emitted_pdb.?.getPath3(b, step);
+ const full_pdb_path = b.getInstallPath(pdb_dir, fs.path.basename(src_path.sub_path));
+ const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_pdb_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_path, full_pdb_path, @errorName(err),
+ src_path.sub_path, full_pdb_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
@@ -164,11 +164,11 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
if (install_artifact.h_dir) |h_dir| {
if (install_artifact.emitted_h) |emitted_h| {
- const full_src_path = emitted_h.getPath2(b, step);
- const full_h_path = b.getInstallPath(h_dir, fs.path.basename(full_src_path));
- const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_h_path, .{}) catch |err| {
+ const src_path = emitted_h.getPath3(b, step);
+ const full_h_path = b.getInstallPath(h_dir, fs.path.basename(src_path.sub_path));
+ const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_h_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_path, full_h_path, @errorName(err),
+ src_path.sub_path, full_h_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
@@ -176,22 +176,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) {
.file => |file| {
- const full_src_path = file.source.getPath2(b, step);
+ const src_path = file.source.getPath3(b, step);
const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path);
- const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_h_path, .{}) catch |err| {
+ const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_h_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_path, full_h_path, @errorName(err),
+ src_path.sub_path, full_h_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
},
.directory => |dir| {
- const full_src_dir_path = dir.source.getPath2(b, step);
+ const src_dir_path = dir.source.getPath3(b, step);
const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path);
- var src_dir = b.build_root.handle.openDir(full_src_dir_path, .{ .iterate = true }) catch |err| {
+ var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.sub_path, .{ .iterate = true }) catch |err| {
return step.fail("unable to open source directory '{s}': {s}", .{
- full_src_dir_path, @errorName(err),
+ src_dir_path.sub_path, @errorName(err),
});
};
defer src_dir.close();
@@ -208,14 +208,15 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
continue :next_entry;
}
}
- const full_src_entry_path = b.pathJoin(&.{ full_src_dir_path, entry.path });
+
+ const src_entry_path = src_dir_path.join(b.allocator, entry.path) catch @panic("OOM");
const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path });
switch (entry.kind) {
.directory => try cwd.makePath(full_dest_path),
.file => {
- const p = fs.Dir.updateFile(cwd, full_src_entry_path, cwd, full_dest_path, .{}) catch |err| {
+ const p = fs.Dir.updateFile(src_entry_path.root_dir.handle, src_entry_path.sub_path, cwd, full_dest_path, .{}) catch |err| {
return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
- full_src_entry_path, full_dest_path, @errorName(err),
+ src_entry_path.sub_path, full_dest_path, @errorName(err),
});
};
all_cached = all_cached and p == .fresh;
lib/std/Build/Step/Run.zig
@@ -7,6 +7,7 @@ const mem = std.mem;
const process = std.process;
const EnvMap = process.EnvMap;
const assert = std.debug.assert;
+const Path = Build.Cache.Path;
const Run = @This();
@@ -93,7 +94,7 @@ cached_test_metadata: ?CachedTestMetadata = null,
/// Populated during the fuzz phase if this run step corresponds to a unit test
/// executable that contains fuzz tests.
-rebuilt_executable: ?[]const u8,
+rebuilt_executable: ?Path,
/// If this Run step was produced by a Compile step, it is tracked here.
producer: ?*Step.Compile,
@@ -872,7 +873,7 @@ pub fn rerunInFuzzMode(
.artifact => |pa| {
const artifact = pa.artifact;
const file_path = if (artifact == run.producer.?)
- run.rebuilt_executable.?
+ b.fmt("{}", .{run.rebuilt_executable.?})
else
(artifact.installed_path orelse artifact.generated_bin.?.path.?);
try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
lib/std/Build/Step/TranslateC.zig
@@ -153,12 +153,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
try argv_list.append(c_macro);
}
- try argv_list.append(translate_c.source.getPath2(b, step));
+ const c_source_path = translate_c.source.getPath2(b, step);
+ try argv_list.append(c_source_path);
- const output_path = try step.evalZigProcess(argv_list.items, prog_node, false);
+ const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false);
- translate_c.out_basename = fs.path.basename(output_path.?);
- const output_dir = fs.path.dirname(output_path.?).?;
-
- translate_c.output_file.path = b.pathJoin(&.{ output_dir, translate_c.out_basename });
+ const basename = std.fs.path.stem(std.fs.path.basename(c_source_path));
+ translate_c.out_basename = b.fmt("{s}.zig", .{basename});
+ translate_c.output_file.path = output_dir.?.joinString(b.allocator, translate_c.out_basename) catch @panic("OOM");
}
lib/std/Build/Cache.zig
@@ -896,8 +896,8 @@ pub const Manifest = struct {
}
}
- /// Returns a hex encoded hash of the inputs.
- pub fn final(self: *Manifest) HexDigest {
+ /// Returns a binary hash of the inputs.
+ pub fn finalBin(self: *Manifest) BinDigest {
assert(self.manifest_file != null);
// We don't close the manifest file yet, because we want to
@@ -908,7 +908,12 @@ pub const Manifest = struct {
var bin_digest: BinDigest = undefined;
self.hash.hasher.final(&bin_digest);
+ return bin_digest;
+ }
+ /// Returns a hex encoded hash of the inputs.
+ pub fn final(self: *Manifest) HexDigest {
+ const bin_digest = self.finalBin();
return binToHex(bin_digest);
}
lib/std/Build/Fuzz.zig
@@ -100,6 +100,15 @@ pub fn start(
}
fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
+ rebuildTestsWorkerRunFallible(run, ttyconf, parent_prog_node) catch |err| {
+ const compile = run.producer.?;
+ log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
+ compile.step.name, @errorName(err),
+ });
+ };
+}
+
+fn rebuildTestsWorkerRunFallible(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) !void {
const gpa = run.step.owner.allocator;
const stderr = std.io.getStdErr();
@@ -121,14 +130,9 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog
const rebuilt_bin_path = result catch |err| switch (err) {
error.MakeFailed => return,
- else => {
- log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
- compile.step.name, @errorName(err),
- });
- return;
- },
+ else => |other| return other,
};
- run.rebuilt_executable = rebuilt_bin_path;
+ run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
}
fn fuzzWorkerRun(
lib/std/Build/Step.zig
@@ -317,6 +317,8 @@ const Build = std.Build;
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const builtin = @import("builtin");
+const Cache = Build.Cache;
+const Path = Cache.Path;
pub fn evalChildProcess(s: *Step, argv: []const []const u8) ![]u8 {
const run_result = try captureChildProcess(s, std.Progress.Node.none, argv);
@@ -373,7 +375,7 @@ pub fn evalZigProcess(
argv: []const []const u8,
prog_node: std.Progress.Node,
watch: bool,
-) !?[]const u8 {
+) !?Path {
if (s.getZigProcess()) |zp| update: {
assert(watch);
if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);
@@ -477,7 +479,7 @@ pub fn evalZigProcess(
return result;
}
-fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
+fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path {
const b = s.owner;
const arena = b.allocator;
@@ -487,7 +489,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
if (!watch) try sendMessage(zp.child.stdin.?, .exit);
const Header = std.zig.Server.Message.Header;
- var result: ?[]const u8 = null;
+ var result: ?Path = null;
const stdout = zp.poller.fifo(.stdout);
@@ -531,16 +533,15 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
break;
}
},
- .emit_bin_path => {
- const EbpHdr = std.zig.Server.Message.EmitBinPath;
+ .emit_digest => {
+ const EbpHdr = std.zig.Server.Message.EmitDigest;
const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
s.result_cached = ebp_hdr.flags.cache_hit;
- result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
- if (watch) {
- // This message indicates the end of the update.
- stdout.discard(body.len);
- break;
- }
+ const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
+ result = Path{
+ .root_dir = b.cache_root,
+ .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
+ };
},
.file_system_inputs => {
s.clearWatchInputs();
lib/std/zig/Server.zig
@@ -14,8 +14,8 @@ pub const Message = struct {
zig_version,
/// Body is an ErrorBundle.
error_bundle,
- /// Body is a EmitBinPath.
- emit_bin_path,
+ /// Body is a EmitDigest.
+ emit_digest,
/// Body is a TestMetadata
test_metadata,
/// Body is a TestResults
@@ -82,8 +82,8 @@ pub const Message = struct {
};
/// Trailing:
- /// * file system path where the emitted binary can be found
- pub const EmitBinPath = extern struct {
+ /// * the hex digest of the cache directory within the /o/ subdirectory.
+ pub const EmitDigest = extern struct {
flags: Flags,
pub const Flags = packed struct(u8) {
@@ -196,17 +196,17 @@ pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void {
}, &.{std.mem.asBytes(&msg_le)});
}
-pub fn serveEmitBinPath(
+pub fn serveEmitDigest(
s: *Server,
- fs_path: []const u8,
- header: OutMessage.EmitBinPath,
+ digest: *const [Cache.bin_digest_len]u8,
+ header: OutMessage.EmitDigest,
) !void {
try s.serveMessage(.{
- .tag = .emit_bin_path,
- .bytes_len = @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath)),
+ .tag = .emit_digest,
+ .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)),
}, &.{
std.mem.asBytes(&header),
- fs_path,
+ digest,
});
}
@@ -328,3 +328,4 @@ const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const native_endian = builtin.target.cpu.arch.endian();
const need_bswap = native_endian != .little;
+const Cache = std.Build.Cache;
lib/std/Build.zig
@@ -2373,7 +2373,7 @@ pub const LazyPath = union(enum) {
// basis for not traversing up too many directories.
var file_path: Cache.Path = .{
- .root_dir = gen.file.step.owner.build_root,
+ .root_dir = Cache.Directory.cwd(),
.sub_path = gen.file.path orelse {
std.debug.lockStdErr();
const stderr = std.io.getStdErr();
src/Compilation.zig
@@ -39,6 +39,8 @@ const Air = @import("Air.zig");
const Builtin = @import("Builtin.zig");
const LlvmObject = @import("codegen/llvm.zig").Object;
const dev = @import("dev.zig");
+pub const Directory = Cache.Directory;
+const Path = Cache.Path;
pub const Config = @import("Compilation/Config.zig");
@@ -269,6 +271,11 @@ llvm_opt_bisect_limit: c_int,
file_system_inputs: ?*std.ArrayListUnmanaged(u8),
+/// This is the digest of the cache for the current compilation.
+/// This digest will be known after update() is called.
+digest: ?[Cache.bin_digest_len]u8 = null,
+
+/// TODO(robin): Remove because it is the same as Cache.Path
pub const Emit = struct {
/// Where the output will go.
directory: Directory,
@@ -868,8 +875,6 @@ pub const LldError = struct {
}
};
-pub const Directory = Cache.Directory;
-
pub const EmitLoc = struct {
/// If this is `null` it means the file will be output to the cache directory.
/// When provided, both the open file handle and the path name must outlive the `Compilation`.
@@ -1672,7 +1677,9 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
// In the case of incremental cache mode, this `artifact_directory`
// is computed based on a hash of non-linker inputs, and it is where all
// build artifacts are stored (even while in-progress).
+ comp.digest = hash.peekBin();
const digest = hash.final();
+
const artifact_sub_dir = "o" ++ std.fs.path.sep_str ++ digest;
var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{});
errdefer artifact_dir.close();
@@ -2121,9 +2128,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
comp.last_update_was_cache_hit = true;
log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});
- const digest = man.final();
+ const bin_digest = man.finalBin();
+ const hex_digest = Cache.binToHex(bin_digest);
- comp.wholeCacheModeSetBinFilePath(whole, &digest);
+ comp.digest = bin_digest;
+ comp.wholeCacheModeSetBinFilePath(whole, &hex_digest);
assert(whole.lock == null);
whole.lock = man.toOwnedLock();
@@ -2329,7 +2338,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
}
- const digest = man.final();
+ const bin_digest = man.finalBin();
+ const hex_digest = Cache.binToHex(bin_digest);
// Rename the temporary directory into place.
// Close tmp dir and link.File to avoid open handle during rename.
@@ -2341,7 +2351,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
const s = std.fs.path.sep_str;
const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(tmp_dir_rand_int);
- const o_sub_path = "o" ++ s ++ digest;
+ const o_sub_path = "o" ++ s ++ hex_digest;
// Work around windows `AccessDenied` if any files within this
// directory are open by closing and reopening the file handles.
@@ -2376,7 +2386,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
},
);
};
- comp.wholeCacheModeSetBinFilePath(whole, &digest);
+ comp.digest = bin_digest;
+ comp.wholeCacheModeSetBinFilePath(whole, &hex_digest);
// The linker flush functions need to know the final output path
// for debug info purposes because executable debug info contains
@@ -2393,9 +2404,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
}
}
- try flush(comp, arena, .main, main_progress_node);
-
- if (try comp.totalErrorCount() != 0) return;
+ try flush(comp, arena, .{
+ .root_dir = comp.local_cache_directory,
+ .sub_path = o_sub_path,
+ }, .main, main_progress_node);
// Failure here only means an unnecessary cache miss.
man.writeManifest() catch |err| {
@@ -2410,8 +2422,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
assert(whole.lock == null);
whole.lock = man.toOwnedLock();
},
- .incremental => {
- try flush(comp, arena, .main, main_progress_node);
+ .incremental => |incremental| {
+ try flush(comp, arena, .{
+ .root_dir = incremental.artifact_directory,
+ }, .main, main_progress_node);
},
}
}
@@ -2440,7 +2454,13 @@ pub fn appendFileSystemInput(
std.debug.panic("missing prefix directory: {}, {s}", .{ root, sub_file_path });
}
-fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) !void {
+fn flush(
+ comp: *Compilation,
+ arena: Allocator,
+ default_artifact_directory: Path,
+ tid: Zcu.PerThread.Id,
+ prog_node: std.Progress.Node,
+) !void {
if (comp.bin_file) |lf| {
// This is needed before reading the error flags.
lf.flush(arena, tid, prog_node) catch |err| switch (err) {
@@ -2454,17 +2474,7 @@ fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node:
try link.File.C.flushEmitH(zcu);
if (zcu.llvm_object) |llvm_object| {
- const default_emit = switch (comp.cache_use) {
- .whole => |whole| .{
- .directory = whole.tmp_artifact_directory.?,
- .sub_path = "dummy",
- },
- .incremental => |incremental| .{
- .directory = incremental.artifact_directory,
- .sub_path = "dummy",
- },
- };
- try emitLlvmObject(comp, arena, default_emit, null, llvm_object, prog_node);
+ try emitLlvmObject(comp, arena, default_artifact_directory, null, llvm_object, prog_node);
}
}
}
@@ -2745,7 +2755,7 @@ fn emitOthers(comp: *Compilation) void {
pub fn emitLlvmObject(
comp: *Compilation,
arena: Allocator,
- default_emit: Emit,
+ default_artifact_directory: Path,
bin_emit_loc: ?EmitLoc,
llvm_object: LlvmObject.Ptr,
prog_node: std.Progress.Node,
@@ -2756,10 +2766,10 @@ pub fn emitLlvmObject(
try llvm_object.emit(.{
.pre_ir_path = comp.verbose_llvm_ir,
.pre_bc_path = comp.verbose_llvm_bc,
- .bin_path = try resolveEmitLoc(arena, default_emit, bin_emit_loc),
- .asm_path = try resolveEmitLoc(arena, default_emit, comp.emit_asm),
- .post_ir_path = try resolveEmitLoc(arena, default_emit, comp.emit_llvm_ir),
- .post_bc_path = try resolveEmitLoc(arena, default_emit, comp.emit_llvm_bc),
+ .bin_path = try resolveEmitLoc(arena, default_artifact_directory, bin_emit_loc),
+ .asm_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_asm),
+ .post_ir_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_llvm_ir),
+ .post_bc_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_llvm_bc),
.is_debug = comp.root_mod.optimize_mode == .Debug,
.is_small = comp.root_mod.optimize_mode == .ReleaseSmall,
@@ -2772,14 +2782,14 @@ pub fn emitLlvmObject(
fn resolveEmitLoc(
arena: Allocator,
- default_emit: Emit,
+ default_artifact_directory: Path,
opt_loc: ?EmitLoc,
) Allocator.Error!?[*:0]const u8 {
const loc = opt_loc orelse return null;
const slice = if (loc.directory) |directory|
try directory.joinZ(arena, &.{loc.basename})
else
- try default_emit.basenamePath(arena, loc.basename);
+ try default_artifact_directory.joinStringZ(arena, loc.basename);
return slice.ptr;
}
@@ -4403,7 +4413,7 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest
}
pub const CImportResult = struct {
- out_zig_path: []u8,
+ digest: [Cache.bin_digest_len]u8,
cache_hit: bool,
errors: std.zig.ErrorBundle,
@@ -4413,8 +4423,6 @@ pub const CImportResult = struct {
};
/// Caller owns returned memory.
-/// This API is currently coupled pretty tightly to stage1's needs; it will need to be reworked
-/// a bit when we want to start using it from self-hosted.
pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module) !CImportResult {
dev.check(.translate_c_command);
@@ -4503,7 +4511,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
error.OutOfMemory => return error.OutOfMemory,
error.SemanticAnalyzeFail => {
return CImportResult{
- .out_zig_path = "",
+ .digest = undefined,
.cache_hit = actual_hit,
.errors = errors,
};
@@ -4528,8 +4536,9 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
.incremental => {},
}
- const digest = man.final();
- const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
+ const bin_digest = man.finalBin();
+ const hex_digest = Cache.binToHex(bin_digest);
+ const o_sub_path = "o" ++ std.fs.path.sep_str ++ hex_digest;
var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
defer o_dir.close();
@@ -4541,8 +4550,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
try out_zig_file.writeAll(formatted);
- break :digest digest;
- } else man.final();
+ break :digest bin_digest;
+ } else man.finalBin();
if (man.have_exclusive_lock) {
// Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is
@@ -4554,14 +4563,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
};
}
- const out_zig_path = try comp.local_cache_directory.join(comp.arena, &.{
- "o", &digest, cimport_zig_basename,
- });
- if (comp.verbose_cimport) {
- log.info("C import output: {s}", .{out_zig_path});
- }
return CImportResult{
- .out_zig_path = out_zig_path,
+ .digest = digest,
.cache_hit = actual_hit,
.errors = std.zig.ErrorBundle.empty,
};
src/link.zig
@@ -1029,7 +1029,10 @@ pub const File = struct {
llvm_object: LlvmObject.Ptr,
prog_node: std.Progress.Node,
) !void {
- return base.comp.emitLlvmObject(arena, base.emit, .{
+ return base.comp.emitLlvmObject(arena, .{
+ .root_dir = base.emit.directory,
+ .sub_path = std.fs.path.dirname(base.emit.sub_path) orelse "",
+ }, .{
.directory = null,
.basename = base.zcu_object_sub_path.?,
}, llvm_object, prog_node);
src/main.zig
@@ -4142,7 +4142,7 @@ fn serve(
if (output.errors.errorMessageCount() != 0) {
try server.serveErrorBundle(output.errors);
} else {
- try server.serveEmitBinPath(output.out_zig_path, .{
+ try server.serveEmitDigest(&output.digest, .{
.flags = .{ .cache_hit = output.cache_hit },
});
}
@@ -4229,62 +4229,10 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
return;
}
- // This logic is counter-intuitive because the protocol accounts for each
- // emitted artifact possibly being in a different location, which correctly
- // matches the behavior of the compiler, however, the build system
- // currently always passes flags that makes all build artifacts output to
- // the same local cache directory, and relies on them all being in the same
- // directory.
- //
- // So, until the build system and protocol are changed to reflect this,
- // this logic must ensure that emit_bin_path is emitted for at least one
- // thing, if there are any artifacts.
-
- switch (comp.cache_use) {
- .incremental => if (comp.bin_file) |lf| {
- const full_path = try lf.emit.directory.join(gpa, &.{lf.emit.sub_path});
- defer gpa.free(full_path);
- try s.serveEmitBinPath(full_path, .{
- .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
- });
- return;
- },
- .whole => |whole| if (whole.bin_sub_path) |sub_path| {
- const full_path = try comp.local_cache_directory.join(gpa, &.{sub_path});
- defer gpa.free(full_path);
- try s.serveEmitBinPath(full_path, .{
- .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
- });
- return;
- },
- }
-
- for ([_]?Compilation.Emit{
- comp.docs_emit,
- comp.implib_emit,
- }) |opt_emit| {
- const emit = opt_emit orelse continue;
- const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
- defer gpa.free(full_path);
- try s.serveEmitBinPath(full_path, .{
+ if (comp.digest) |digest| {
+ try s.serveEmitDigest(&digest, .{
.flags = .{ .cache_hit = comp.last_update_was_cache_hit },
});
- return;
- }
-
- for ([_]?Compilation.EmitLoc{
- comp.emit_asm,
- comp.emit_llvm_ir,
- comp.emit_llvm_bc,
- }) |opt_emit_loc| {
- const emit_loc = opt_emit_loc orelse continue;
- const directory = emit_loc.directory orelse continue;
- const full_path = try directory.join(gpa, &.{emit_loc.basename});
- defer gpa.free(full_path);
- try s.serveEmitBinPath(full_path, .{
- .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
- });
- return;
}
// Serve empty error bundle to indicate the update is done.
@@ -4539,9 +4487,11 @@ fn cmdTranslateC(
};
if (fancy_output) |p| p.cache_hit = true;
- const digest = if (try man.hit()) digest: {
+ const bin_digest, const hex_digest = if (try man.hit()) digest: {
if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
- break :digest man.final();
+ const bin_digest = man.finalBin();
+ const hex_digest = Cache.binToHex(bin_digest);
+ break :digest .{ bin_digest, hex_digest };
} else digest: {
if (fancy_output) |p| p.cache_hit = false;
var argv = std.ArrayList([]const u8).init(arena);
@@ -4639,8 +4589,10 @@ fn cmdTranslateC(
};
}
- const digest = man.final();
- const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest });
+ const bin_digest = man.finalBin();
+ const hex_digest = Cache.binToHex(bin_digest);
+
+ const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &hex_digest });
var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
defer o_dir.close();
@@ -4656,16 +4608,14 @@ fn cmdTranslateC(
if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
- break :digest digest;
+ break :digest .{ bin_digest, hex_digest };
};
if (fancy_output) |p| {
- p.out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{
- "o", &digest, translated_zig_basename,
- });
+ p.digest = bin_digest;
p.errors = std.zig.ErrorBundle.empty;
} else {
- const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename });
+ const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &hex_digest, translated_zig_basename });
const zig_file = comp.local_cache_directory.handle.openFile(out_zig_path, .{}) catch |err| {
const path = comp.local_cache_directory.path orelse ".";
fatal("unable to open cached translated zig file '{s}{s}{s}': {s}", .{ path, fs.path.sep_str, out_zig_path, @errorName(err) });
src/Sema.zig
@@ -183,6 +183,7 @@ const InternPool = @import("InternPool.zig");
const Alignment = InternPool.Alignment;
const AnalUnit = InternPool.AnalUnit;
const ComptimeAllocIndex = InternPool.ComptimeAllocIndex;
+const Cache = std.Build.Cache;
pub const default_branch_quota = 1000;
pub const default_reference_trace_len = 2;
@@ -5871,16 +5872,18 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr
return sema.failWithOwnedErrorMsg(&child_block, msg);
}
const parent_mod = parent_block.ownerModule();
+ const digest = Cache.binToHex(c_import_res.digest);
+ const c_import_zig_path = try comp.arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ digest);
const c_import_mod = Package.Module.create(comp.arena, .{
.global_cache_directory = comp.global_cache_directory,
.paths = .{
.root = .{
- .root_dir = Compilation.Directory.cwd(),
- .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "",
+ .root_dir = comp.local_cache_directory,
+ .sub_path = c_import_zig_path,
},
- .root_src_path = std.fs.path.basename(c_import_res.out_zig_path),
+ .root_src_path = "cimport.zig",
},
- .fully_qualified_name = c_import_res.out_zig_path,
+ .fully_qualified_name = c_import_zig_path,
.cc_argv = parent_mod.cc_argv,
.inherited = .{},
.global = comp.config,
tools/incr-check.zig
@@ -1,6 +1,7 @@
const std = @import("std");
const fatal = std.process.fatal;
const Allocator = std.mem.Allocator;
+const Cache = std.Build.Cache;
const usage = "usage: incr-check <zig binary path> <input file> [--zig-lib-dir lib] [--debug-zcu] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]";
@@ -233,30 +234,52 @@ const Eval = struct {
fatal("error_bundle included unexpected stderr:\n{s}", .{stderr_data});
}
}
- if (result_error_bundle.errorMessageCount() == 0) {
- // Empty bundle indicates successful update in a `-fno-emit-bin` build.
- try eval.checkSuccessOutcome(update, null, prog_node);
- } else {
+ if (result_error_bundle.errorMessageCount() != 0) {
try eval.checkErrorOutcome(update, result_error_bundle);
}
// This message indicates the end of the update.
stdout.discard(body.len);
return;
},
- .emit_bin_path => {
- const EbpHdr = std.zig.Server.Message.EmitBinPath;
+ .emit_digest => {
+ const EbpHdr = std.zig.Server.Message.EmitDigest;
const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
_ = ebp_hdr;
- const result_binary = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
if (stderr.readableLength() > 0) {
const stderr_data = try stderr.toOwnedSlice();
if (eval.allow_stderr) {
- std.log.info("emit_bin_path included stderr:\n{s}", .{stderr_data});
+ std.log.info("emit_digest included stderr:\n{s}", .{stderr_data});
} else {
- fatal("emit_bin_path included unexpected stderr:\n{s}", .{stderr_data});
+ fatal("emit_digest included unexpected stderr:\n{s}", .{stderr_data});
}
}
- try eval.checkSuccessOutcome(update, result_binary, prog_node);
+
+ if (eval.emit == .none) {
+ try eval.checkSuccessOutcome(update, null, prog_node);
+ // This message indicates the end of the update.
+ stdout.discard(body.len);
+ return;
+ }
+
+ const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
+ const result_dir = ".local-cache" ++ std.fs.path.sep_str ++ "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*);
+
+ const name = std.fs.path.stem(std.fs.path.basename(eval.case.root_source_file));
+ const bin_name = try std.zig.binNameAlloc(arena, .{
+ .root_name = name,
+ .target = try std.zig.system.resolveTargetQuery(try std.Build.parseTargetQuery(.{
+ .arch_os_abi = eval.case.target_query,
+ .object_format = switch (eval.emit) {
+ .none => unreachable,
+ .bin => null,
+ .c => "c",
+ },
+ })),
+ .output_mode = .Exe,
+ });
+ const bin_path = try std.fs.path.join(arena, &.{ result_dir, bin_name });
+
+ try eval.checkSuccessOutcome(update, bin_path, prog_node);
// This message indicates the end of the update.
stdout.discard(body.len);
return;