Commit d3c9bfada6

@@ -1,55 +1,117 @@
-const std = @import("../std.zig");
-const Step = std.Build.Step;
-const fs = std.fs;
-const ArrayList = std.ArrayList;
-
-const WriteFileStep = @This();
-
-pub const base_id = .write_file;
+//! WriteFileStep is primarily used to create a directory in an appropriate
+//! location inside the local cache which has a set of files that have either
+//! been generated during the build, or are copied from the source package.
+//!
+//! However, this step has an additional capability of writing data to paths
+//! relative to the package root, effectively mutating the package's source
+//! files. Be careful with the latter functionality; it should not be used
+//! during the normal build process, but as a utility run by a developer with
+//! intention to update source files, which will then be committed to version
+//! control.
 
 step: Step,
 builder: *std.Build,
-files: std.TailQueue(File),
+/// The elements here are pointers because we need stable pointers for the
+/// GeneratedFile field.
+files: std.ArrayListUnmanaged(*File),
+output_source_files: std.ArrayListUnmanaged(OutputSourceFile),
+
+pub const base_id = .write_file;
 
 pub const File = struct {
-    source: std.Build.GeneratedFile,
-    basename: []const u8,
+    generated_file: std.Build.GeneratedFile,
+    sub_path: []const u8,
+    contents: Contents,
+};
+
+pub const OutputSourceFile = struct {
+    contents: Contents,
+    sub_path: []const u8,
+};
+
+pub const Contents = union(enum) {
     bytes: []const u8,
+    copy: std.Build.FileSource,
 };
 
 pub fn init(builder: *std.Build) WriteFileStep {
-    return WriteFileStep{
+    return .{
         .builder = builder,
         .step = Step.init(.write_file, "writefile", builder.allocator, make),
         .files = .{},
+        .output_source_files = .{},
     };
 }
 
-pub fn add(self: *WriteFileStep, basename: []const u8, bytes: []const u8) void {
-    const node = self.builder.allocator.create(std.TailQueue(File).Node) catch @panic("unhandled error");
-    node.* = .{
-        .data = .{
-            .source = std.Build.GeneratedFile{ .step = &self.step },
-            .basename = self.builder.dupePath(basename),
-            .bytes = self.builder.dupe(bytes),
-        },
+pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
+    const gpa = wf.builder.allocator;
+    const file = gpa.create(File) catch @panic("OOM");
+    file.* = .{
+        .generated_file = .{ .step = &wf.step },
+        .sub_path = wf.builder.dupePath(sub_path),
+        .contents = .{ .bytes = wf.builder.dupe(bytes) },
+    };
+    wf.files.append(gpa, file) catch @panic("OOM");
+}
+
+/// Place the file into the generated directory within the local cache,
+/// along with all the rest of the files added to this step. The parameter
+/// here is the destination path relative to the local cache directory
+/// associated with this WriteFileStep. It may be a basename, or it may
+/// include sub-directories, in which case this step will ensure the
+/// required sub-path exists.
+/// This is the option expected to be used most commonly with `addCopyFile`.
+pub fn addCopyFile(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
+    const gpa = wf.builder.allocator;
+    const file = gpa.create(File) catch @panic("OOM");
+    file.* = .{
+        .generated_file = .{ .step = &wf.step },
+        .sub_path = wf.builder.dupePath(sub_path),
+        .contents = .{ .copy = source },
     };
+    wf.files.append(gpa, file) catch @panic("OOM");
+}
 
-    self.files.append(node);
+/// A path relative to the package root.
+/// Be careful with this because it updates source files. This should not be
+/// used as part of the normal build process, but as a utility occasionally
+/// run by a developer with intent to modify source files and then commit
+/// those changes to version control.
+/// A file added this way is not available with `getFileSource`.
+pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub_path: []const u8) void {
+    wf.output_source_files.append(wf.builder.allocator, .{
+        .contents = .{ .copy = source },
+        .sub_path = sub_path,
+    }) catch @panic("OOM");
 }
 
-/// Gets a file source for the given basename. If the file does not exist, returns `null`.
-pub fn getFileSource(step: *WriteFileStep, basename: []const u8) ?std.Build.FileSource {
-    var it = step.files.first;
-    while (it) |node| : (it = node.next) {
-        if (std.mem.eql(u8, node.data.basename, basename))
-            return std.Build.FileSource{ .generated = &node.data.source };
+/// Gets a file source for the given sub_path. If the file does not exist, returns `null`.
+pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSource {
+    for (wf.files.items) |file| {
+        if (std.mem.eql(u8, file.sub_path, sub_path)) {
+            return .{ .generated = &file.generated_file };
+        }
     }
     return null;
 }
 
 fn make(step: *Step) !void {
-    const self = @fieldParentPtr(WriteFileStep, "step", step);
+    const wf = @fieldParentPtr(WriteFileStep, "step", step);
+
+    // Writing to source files is kind of an extra capability of this
+    // WriteFileStep - arguably it should be a different step. But anyway here
+    // it is, it happens unconditionally and does not interact with the other
+    // files here.
+    for (wf.output_source_files.items) |output_source_file| {
+        const basename = fs.path.basename(output_source_file.sub_path);
+        if (fs.path.dirname(output_source_file.sub_path)) |dirname| {
+            var dir = try wf.builder.build_root.handle.makeOpenPath(dirname, .{});
+            defer dir.close();
+            try writeFile(wf, dir, output_source_file.contents, basename);
+        } else {
+            try writeFile(wf, wf.builder.build_root.handle, output_source_file.contents, basename);
+        }
+    }
 
     // The cache is used here not really as a way to speed things up - because writing
     // the data to a file would probably be very fast - but as a way to find a canonical
@@ -58,56 +120,96 @@ fn make(step: *Step) !void {
     // If, for example, a hard-coded path was used as the location to put WriteFileStep
     // files, then two WriteFileSteps executing in parallel might clobber each other.
 
-    // TODO port the cache system from the compiler to zig std lib. Until then
-    // we directly construct the path, and no "cache hit" detection happens;
-    // the files are always written.
-    // Note there is similar code over in ConfigHeaderStep.
-    const Hasher = std.crypto.auth.siphash.SipHash128(1, 3);
+    var man = wf.builder.cache.obtain();
+    defer man.deinit();
+
     // Random bytes to make WriteFileStep unique. Refresh this with
     // new random bytes when WriteFileStep implementation is modified
     // in a non-backwards-compatible way.
-    var hash = Hasher.init("eagVR1dYXoE7ARDP");
-
-    {
-        var it = self.files.first;
-        while (it) |node| : (it = node.next) {
-            hash.update(node.data.basename);
-            hash.update(node.data.bytes);
-            hash.update("|");
+    man.hash.add(@as(u32, 0xd767ee59));
+
+    for (wf.files.items) |file| {
+        man.hash.addBytes(file.sub_path);
+        switch (file.contents) {
+            .bytes => |bytes| {
+                man.hash.addBytes(bytes);
+            },
+            .copy => |file_source| {
+                _ = try man.addFile(file_source.getPath(wf.builder), null);
+            },
         }
     }
-    var digest: [16]u8 = undefined;
-    hash.final(&digest);
-    var hash_basename: [digest.len * 2]u8 = undefined;
-    _ = std.fmt.bufPrint(
-        &hash_basename,
-        "{s}",
-        .{std.fmt.fmtSliceHexLower(&digest)},
-    ) catch unreachable;
-
-    const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
-        "o", &hash_basename,
-    });
-    var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
-        std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
+
+    if (man.hit() catch |err| failWithCacheError(man, err)) {
+        // Cache hit, skip writing file data.
+        const digest = man.final();
+        for (wf.files.items) |file| {
+            file.generated_file.path = try wf.builder.cache_root.join(
+                wf.builder.allocator,
+                &.{ "o", &digest, file.sub_path },
+            );
+        }
+        return;
+    }
+
+    const digest = man.final();
+    const cache_path = "o" ++ fs.path.sep_str ++ digest;
+
+    var cache_dir = wf.builder.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
+        std.debug.print("unable to make path {s}: {s}\n", .{ cache_path, @errorName(err) });
         return err;
     };
-    defer dir.close();
-    {
-        var it = self.files.first;
-        while (it) |node| : (it = node.next) {
-            dir.writeFile(node.data.basename, node.data.bytes) catch |err| {
-                std.debug.print("unable to write {s} into {s}: {s}\n", .{
-                    node.data.basename,
-                    output_dir,
-                    @errorName(err),
-                });
-                return err;
-            };
-            node.data.source.path = try fs.path.join(
-                self.builder.allocator,
-                &[_][]const u8{ output_dir, node.data.basename },
-            );
+    defer cache_dir.close();
+
+    for (wf.files.items) |file| {
+        const basename = fs.path.basename(file.sub_path);
+        if (fs.path.dirname(file.sub_path)) |dirname| {
+            var dir = try wf.builder.cache_root.handle.makeOpenPath(dirname, .{});
+            defer dir.close();
+            try writeFile(wf, dir, file.contents, basename);
+        } else {
+            try writeFile(wf, cache_dir, file.contents, basename);
         }
+
+        file.generated_file.path = try wf.builder.cache_root.join(
+            wf.builder.allocator,
+            &.{ cache_path, file.sub_path },
+        );
     }
+
+    try man.writeManifest();
 }
+
+fn writeFile(wf: *WriteFileStep, dir: fs.Dir, contents: Contents, basename: []const u8) !void {
+    // TODO after landing concurrency PR, improve error reporting here
+    switch (contents) {
+        .bytes => |bytes| return dir.writeFile(basename, bytes),
+        .copy => |file_source| {
+            const source_path = file_source.getPath(wf.builder);
+            const prev_status = try fs.Dir.updateFile(fs.cwd(), source_path, dir, basename, .{});
+            _ = prev_status; // TODO logging (affected by open PR regarding concurrency)
+        },
+    }
+}
+
+/// TODO consolidate this with the same function in RunStep?
+/// Also properly deal with concurrency (see open PR)
+fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
+    const i = man.failed_file_index orelse failWithSimpleError(err);
+    const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
+    const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
+    std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
+    std.process.exit(1);
+}
+
+fn failWithSimpleError(err: anyerror) noreturn {
+    std.debug.print("{s}\n", .{@errorName(err)});
+    std.process.exit(1);
+}
+
+const std = @import("../std.zig");
+const Step = std.Build.Step;
+const fs = std.fs;
+const ArrayList = std.ArrayList;
+
+const WriteFileStep = @This();

@@ -509,35 +509,8 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
     run_opt.addArg("-o");
     run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
 
-    const CopyFileStep = struct {
-        const Step = std.Build.Step;
-        const FileSource = std.Build.FileSource;
-        const CopyFileStep = @This();
-
-        step: Step,
-        builder: *std.Build,
-        source: FileSource,
-        dest_rel_path: []const u8,
-
-        pub fn init(builder: *std.Build, source: FileSource, dest_rel_path: []const u8) CopyFileStep {
-            return CopyFileStep{
-                .builder = builder,
-                .step = Step.init(.custom, builder.fmt("install {s} to {s}", .{ source.getDisplayName(), dest_rel_path }), builder.allocator, make),
-                .source = source.dupe(builder),
-                .dest_rel_path = builder.dupePath(dest_rel_path),
-            };
-        }
-
-        fn make(step: *Step) !void {
-            const self = @fieldParentPtr(CopyFileStep, "step", step);
-            const full_src_path = self.source.getPath(self.builder);
-            const full_dest_path = self.builder.pathFromRoot(self.dest_rel_path);
-            try self.builder.updateFile(full_src_path, full_dest_path);
-        }
-    };
-
-    const copy_zig_h = try b.allocator.create(CopyFileStep);
-    copy_zig_h.* = CopyFileStep.init(b, .{ .path = "lib/zig.h" }, "stage1/zig.h");
+    const copy_zig_h = b.addWriteFiles();
+    copy_zig_h.addCopyFileToSource(.{ .path = "lib/zig.h" }, "stage1/zig.h");
 
     const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");
     update_zig1_step.dependOn(&run_opt.step);