Commit d97042ad2e
Changed files (9)
lib/std/Build/CompileStep.zig
@@ -83,7 +83,7 @@ max_memory: ?u64 = null,
shared_memory: bool = false,
global_base: ?u64 = null,
c_std: std.Build.CStd,
-override_lib_dir: ?[]const u8,
+zig_lib_dir: ?[]const u8,
main_pkg_path: ?[]const u8,
exec_cmd_args: ?[]const ?[]const u8,
name_prefix: []const u8,
@@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
.installed_headers = ArrayList(*Step).init(builder.allocator),
.object_src = undefined,
.c_std = std.Build.CStd.C99,
- .override_lib_dir = null,
+ .zig_lib_dir = null,
.main_pkg_path = null,
.exec_cmd_args = null,
.name_prefix = "",
@@ -857,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
}
pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
- self.override_lib_dir = self.builder.dupePath(dir_path);
+ self.zig_lib_dir = self.builder.dupePath(dir_path);
}
pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
@@ -1350,10 +1350,10 @@ fn make(step: *Step) !void {
}
try zig_args.append("--cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.cache_root));
+ try zig_args.append(builder.pathFromRoot(builder.cache_root.path orelse "."));
try zig_args.append("--global-cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.global_cache_root));
+ try zig_args.append(builder.pathFromRoot(builder.global_cache_root.path orelse "."));
try zig_args.append("--name");
try zig_args.append(self.name);
@@ -1703,12 +1703,12 @@ fn make(step: *Step) !void {
try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
try addFlag(&zig_args, "build-id", self.build_id);
- if (self.override_lib_dir) |dir| {
+ if (self.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
try zig_args.append(builder.pathFromRoot(dir));
- } else if (builder.override_lib_dir) |dir| {
+ } else if (builder.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
- try zig_args.append(builder.pathFromRoot(dir));
+ try zig_args.append(dir);
}
if (self.main_pkg_path) |dir| {
@@ -1745,23 +1745,15 @@ fn make(step: *Step) !void {
args_length += arg.len + 1; // +1 to account for null terminator
}
if (args_length >= 30 * 1024) {
- const args_dir = try fs.path.join(
- builder.allocator,
- &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
- );
- try std.fs.cwd().makePath(args_dir);
-
- var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
- defer args_arena.deinit();
+ try builder.cache_root.handle.makePath("args");
const args_to_escape = zig_args.items[2..];
- var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);
-
+ var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
arg_blk: for (args_to_escape) |arg| {
for (arg) |c, arg_idx| {
if (c == '\\' or c == '"') {
// Slow path for arguments that need to be escaped. We'll need to allocate and copy
- var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);
+ var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
const writer = escaped.writer();
try writer.writeAll(arg[0..arg_idx]);
for (arg[arg_idx..]) |to_escape| {
@@ -1789,11 +1781,16 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&args_hash)},
);
- const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
- try std.fs.cwd().writeFile(args_file, args);
+ const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
+ try builder.cache_root.handle.writeFile(args_file, args);
+
+ const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
+ "@",
+ builder.pathFromRoot(try builder.cache_root.join(builder.allocator, &.{args_file})),
+ });
zig_args.shrinkRetainingCapacity(2);
- try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
+ try zig_args.append(resolved_args_file);
}
const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);
lib/std/Build/ConfigHeaderStep.zig
@@ -208,9 +208,7 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- const output_dir = try std.fs.path.join(gpa, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
- });
+ const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
// If output_path has directory parts, deal with them. Example:
// output_dir is zig-cache/o/HASH
lib/std/Build/OptionsStep.zig
@@ -234,26 +234,20 @@ fn make(step: *Step) !void {
);
}
- const options_directory = self.builder.pathFromRoot(
- try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ self.builder.cache_root, "options" },
- ),
- );
-
- try fs.cwd().makePath(options_directory);
+ var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
+ defer options_dir.close();
- const options_file = try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ options_directory, &self.hashContentsToFileName() },
- );
+ const basename = self.hashContentsToFileName();
- try fs.cwd().writeFile(options_file, self.contents.items);
+ try options_dir.writeFile(&basename, self.contents.items);
- self.generated_file.path = options_file;
+ self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "options", &basename,
+ });
}
fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
+ // TODO update to use the cache system instead of this
// This implementation is copied from `WriteFileStep.make`
var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
lib/std/Build/RunStep.zig
@@ -44,6 +44,10 @@ print: bool,
/// running if all output files are up-to-date.
condition: enum { output_outdated, always } = .output_outdated,
+/// Additional file paths relative to build.zig that, when modified, indicate
+/// that the RunStep should be re-executed.
+extra_file_dependencies: []const []const u8 = &.{},
+
pub const StdIoAction = union(enum) {
inherit,
ignore,
@@ -184,63 +188,104 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
}
fn needOutputCheck(self: RunStep) bool {
- switch (self.condition) {
- .always => return false,
- .output_outdated => {
- for (self.argv.items) |arg| switch (arg) {
- .output => return true,
- else => continue,
- };
- return false;
- },
- }
+ if (self.extra_file_dependencies.len > 0) return true;
+
+ for (self.argv.items) |arg| switch (arg) {
+ .output => return true,
+ else => continue,
+ };
+
+ return switch (self.condition) {
+ .always => false,
+ .output_outdated => true,
+ };
}
fn make(step: *Step) !void {
const self = @fieldParentPtr(RunStep, "step", step);
+ const need_output_check = self.needOutputCheck();
var argv_list = ArrayList([]const u8).init(self.builder.allocator);
+ var output_placeholders = ArrayList(struct {
+ index: usize,
+ output: Arg.Output,
+ }).init(self.builder.allocator);
+
+ var man = self.builder.cache.obtain();
+ defer man.deinit();
for (self.argv.items) |arg| {
switch (arg) {
- .bytes => |bytes| try argv_list.append(bytes),
- .file_source => |file| try argv_list.append(file.getPath(self.builder)),
+ .bytes => |bytes| {
+ try argv_list.append(bytes);
+ man.hash.addBytes(bytes);
+ },
+ .file_source => |file| {
+ const file_path = file.getPath(self.builder);
+ try argv_list.append(file_path);
+ _ = try man.addFile(file_path, null);
+ },
.artifact => |artifact| {
if (artifact.target.isWindows()) {
// On Windows we don't have rpaths so we have to add .dll search paths to PATH
self.addPathForDynLibs(artifact);
}
- const executable_path = artifact.installed_path orelse
+ const file_path = artifact.installed_path orelse
artifact.getOutputSource().getPath(self.builder);
- try argv_list.append(executable_path);
+
+ try argv_list.append(file_path);
+
+ _ = try man.addFile(file_path, null);
},
.output => |output| {
- // TODO: until the cache system is brought into the build system,
- // we use a temporary directory here for each run.
- var digest: [16]u8 = undefined;
- std.crypto.random.bytes(&digest);
- var hash_basename: [digest.len * 2]u8 = undefined;
- _ = std.fmt.bufPrint(
- &hash_basename,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&digest)},
- ) catch unreachable;
-
- const output_path = try fs.path.join(self.builder.allocator, &[_][]const u8{
- self.builder.cache_root, "tmp", &hash_basename, output.basename,
+ man.hash.addBytes(output.basename);
+ // Add a placeholder into the argument list because we need the
+ // manifest hash to be updated with all arguments before the
+ // object directory is computed.
+ try argv_list.append("");
+ try output_placeholders.append(.{
+ .index = argv_list.items.len - 1,
+ .output = output,
});
- const output_dir = fs.path.dirname(output_path).?;
- fs.cwd().makePath(output_dir) catch |err| {
- std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
- return err;
- };
-
- output.generated_file.path = output_path;
- try argv_list.append(output_path);
},
}
}
+ if (need_output_check) {
+ for (self.extra_file_dependencies) |file_path| {
+ _ = try man.addFile(self.builder.pathFromRoot(file_path), null);
+ }
+
+ if (man.hit() catch |err| failWithCacheError(man, err)) {
+ // cache hit, skip running command
+ const digest = man.final();
+ for (output_placeholders.items) |placeholder| {
+ placeholder.output.generated_file.path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ }
+ return;
+ }
+
+ const digest = man.final();
+
+ for (output_placeholders.items) |placeholder| {
+ const output_path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ const output_dir = fs.path.dirname(output_path).?;
+ fs.cwd().makePath(output_dir) catch |err| {
+ std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
+ return err;
+ };
+
+ placeholder.output.generated_file.path = output_path;
+ argv_list.items[placeholder.index] = output_path;
+ }
+ }
+
try runCommand(
argv_list.items,
self.builder,
@@ -252,6 +297,10 @@ fn make(step: *Step) !void {
self.cwd,
self.print,
);
+
+ if (need_output_check) {
+ try man.writeManifest();
+ }
}
pub fn runCommand(
@@ -265,11 +314,13 @@ pub fn runCommand(
maybe_cwd: ?[]const u8,
print: bool,
) !void {
- const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root;
+ const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path;
if (!std.process.can_spawn) {
const cmd = try std.mem.join(builder.allocator, " ", argv);
- std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
+ std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
+ @tagName(builtin.os.tag), cmd,
+ });
builder.allocator.free(cmd);
return ExecError.ExecNotSupported;
}
@@ -410,6 +461,19 @@ pub fn runCommand(
}
}
+fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
+ const i = man.failed_file_index orelse failWithSimpleError(err);
+ const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
+ const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
+ std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
+ std.process.exit(1);
+}
+
+fn failWithSimpleError(err: anyerror) noreturn {
+ std.debug.print("{s}\n", .{@errorName(err)});
+ std.process.exit(1);
+}
+
fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});
for (argv) |arg| {
lib/std/Build/WriteFileStep.zig
@@ -85,8 +85,8 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
+ const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "o", &hash_basename,
});
var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
lib/std/Build.zig
@@ -79,11 +79,12 @@ search_prefixes: ArrayList([]const u8),
libc_file: ?[]const u8 = null,
installed_files: ArrayList(InstalledFile),
/// Path to the directory containing build.zig.
-build_root: []const u8,
-cache_root: []const u8,
-global_cache_root: []const u8,
-/// zig lib dir
-override_lib_dir: ?[]const u8,
+build_root: Cache.Directory,
+cache_root: Cache.Directory,
+global_cache_root: Cache.Directory,
+cache: *Cache,
+/// If non-null, overrides the default zig lib dir.
+zig_lib_dir: ?[]const u8,
vcpkg_root: VcpkgRoot = .unattempted,
pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
args: ?[][]const u8 = null,
@@ -187,10 +188,11 @@ pub const DirList = struct {
pub fn create(
allocator: Allocator,
zig_exe: []const u8,
- build_root: []const u8,
- cache_root: []const u8,
- global_cache_root: []const u8,
+ build_root: Cache.Directory,
+ cache_root: Cache.Directory,
+ global_cache_root: Cache.Directory,
host: NativeTargetInfo,
+ cache: *Cache,
) !*Build {
const env_map = try allocator.create(EnvMap);
env_map.* = try process.getEnvMap(allocator);
@@ -199,8 +201,9 @@ pub fn create(
self.* = Build{
.zig_exe = zig_exe,
.build_root = build_root,
- .cache_root = try fs.path.relative(allocator, build_root, cache_root),
+ .cache_root = cache_root,
.global_cache_root = global_cache_root,
+ .cache = cache,
.verbose = false,
.verbose_link = false,
.verbose_cc = false,
@@ -232,7 +235,7 @@ pub fn create(
.step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
.description = "Remove build artifacts from prefix path",
},
- .override_lib_dir = null,
+ .zig_lib_dir = null,
.install_path = undefined,
.args = null,
.host = host,
@@ -247,7 +250,7 @@ pub fn create(
fn createChild(
parent: *Build,
dep_name: []const u8,
- build_root: []const u8,
+ build_root: Cache.Directory,
args: anytype,
) !*Build {
const child = try createChildOnly(parent, dep_name, build_root);
@@ -255,7 +258,7 @@ fn createChild(
return child;
}
-fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build {
+fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build {
const allocator = parent.allocator;
const child = try allocator.create(Build);
child.* = .{
@@ -299,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)
.build_root = build_root,
.cache_root = parent.cache_root,
.global_cache_root = parent.global_cache_root,
- .override_lib_dir = parent.override_lib_dir,
+ .cache = parent.cache,
+ .zig_lib_dir = parent.zig_lib_dir,
.debug_log_scopes = parent.debug_log_scopes,
.debug_compile_errors = parent.debug_compile_errors,
.enable_darling = parent.enable_darling,
@@ -381,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
_ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch
unreachable;
- const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename });
+ const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename });
b.resolveInstallPrefix(install_prefix, .{});
}
@@ -398,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });
} else {
self.install_prefix = install_prefix orelse
- (self.pathJoin(&.{ self.build_root, "zig-out" }));
+ (self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error"));
self.install_path = self.install_prefix;
}
@@ -698,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS
}
pub fn make(self: *Build, step_names: []const []const u8) !void {
- try self.makePath(self.cache_root);
-
var wanted_steps = ArrayList(*Step).init(self.allocator);
defer wanted_steps.deinit();
@@ -1225,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,
}
}
-pub fn makePath(self: *Build, path: []const u8) !void {
- fs.cwd().makePath(self.pathFromRoot(path)) catch |err| {
- log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) });
- return err;
- };
-}
-
pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
}
@@ -1346,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {
src_file.close();
}
-pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 {
- return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM");
+pub fn pathFromRoot(b: *Build, p: []const u8) []u8 {
+ return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM");
}
pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {
@@ -1568,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
fn dependencyInner(
b: *Build,
name: []const u8,
- build_root: []const u8,
+ build_root_string: []const u8,
comptime build_zig: type,
args: anytype,
) *Dependency {
+ const build_root: std.Build.Cache.Directory = .{
+ .path = build_root_string,
+ .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
+ std.debug.print("unable to open '{s}': {s}\n", .{
+ build_root_string, @errorName(err),
+ });
+ std.process.exit(1);
+ },
+ };
const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");
sub_builder.runBuild(build_zig) catch @panic("unhandled error");
lib/build_runner.zig
@@ -43,13 +43,40 @@ pub fn main() !void {
const host = try std.zig.system.NativeTargetInfo.detect(.{});
+ const build_root_directory: std.Build.Cache.Directory = .{
+ .path = build_root,
+ .handle = try std.fs.cwd().openDir(build_root, .{}),
+ };
+
+ const local_cache_directory: std.Build.Cache.Directory = .{
+ .path = try std.fs.path.relative(allocator, build_root, cache_root),
+ .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}),
+ };
+
+ const global_cache_directory: std.Build.Cache.Directory = .{
+ .path = try std.fs.path.relative(allocator, build_root, global_cache_root),
+ .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}),
+ };
+
+ var cache: std.Build.Cache = .{
+ .gpa = allocator,
+ .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
+ cache.addPrefix(build_root_directory);
+ cache.addPrefix(local_cache_directory);
+ cache.addPrefix(global_cache_directory);
+
+ //cache.hash.addBytes(builtin.zig_version);
+
const builder = try std.Build.create(
allocator,
zig_exe,
- build_root,
- cache_root,
- global_cache_root,
+ build_root_directory,
+ local_cache_directory,
+ global_cache_directory,
host,
+ &cache,
);
defer builder.destroy();
@@ -138,7 +165,7 @@ pub fn main() !void {
return usageAndErr(builder, false, stderr_stream);
};
} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
- builder.override_lib_dir = nextArg(args, &arg_idx) orelse {
+ builder.zig_lib_dir = nextArg(args, &arg_idx) orelse {
std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});
return usageAndErr(builder, false, stderr_stream);
};
test/tests.zig
@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
const run_cmd = exe.run();
run_cmd.addArgs(&[_][]const u8{
fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,
- b.pathFromRoot(b.cache_root),
+ b.pathFromRoot(b.cache_root.path orelse "."),
});
step.dependOn(&run_cmd.step);
@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {
}
var zig_args = ArrayList([]const u8).init(b.allocator);
- const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable;
+ const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable;
zig_args.append(rel_zig_exe) catch unreachable;
zig_args.append("build") catch unreachable;
build.zig
@@ -40,11 +40,8 @@ pub fn build(b: *std.Build) !void {
});
docgen_exe.single_threaded = single_threaded;
- const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe);
- const langref_out_path = fs.path.join(
- b.allocator,
- &[_][]const u8{ b.cache_root, "langref.html" },
- ) catch unreachable;
+ const rel_zig_exe = try b.build_root.join(b.allocator, &.{b.zig_exe});
+ const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"});
const docgen_cmd = docgen_exe.run();
docgen_cmd.addArgs(&[_][]const u8{
"--zig",
@@ -215,7 +212,7 @@ pub fn build(b: *std.Build) !void {
var code: u8 = undefined;
const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{
- "git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags",
+ "git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags",
}, &code, .Ignore) catch {
break :v version_string;
};