Commit d0bcc390e8
Changed files (12)
lib/std/Build/Step/Compile.zig
@@ -68,7 +68,7 @@ c_std: std.Build.CStd,
/// Set via options; intended to be read-only after that.
zig_lib_dir: ?LazyPath,
/// Set via options; intended to be read-only after that.
-main_pkg_path: ?LazyPath,
+main_mod_path: ?LazyPath,
exec_cmd_args: ?[]const ?[]const u8,
filter: ?[]const u8,
test_evented_io: bool = false,
@@ -316,6 +316,9 @@ pub const Options = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -480,7 +483,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
.installed_headers = ArrayList(*Step).init(owner.allocator),
.c_std = std.Build.CStd.C99,
.zig_lib_dir = null,
- .main_pkg_path = null,
+ .main_mod_path = null,
.exec_cmd_args = null,
.filter = options.filter,
.test_runner = options.test_runner,
@@ -515,8 +518,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
lp.addStepDependencies(&self.step);
}
- if (options.main_pkg_path) |lp| {
- self.main_pkg_path = lp.dupe(self.step.owner);
+ if (options.main_mod_path orelse options.main_pkg_path) |lp| {
+ self.main_mod_path = lp.dupe(self.step.owner);
lp.addStepDependencies(&self.step);
}
@@ -1998,8 +2001,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
try zig_args.append(dir.getPath(b));
}
- if (self.main_pkg_path) |dir| {
- try zig_args.append("--main-pkg-path");
+ if (self.main_mod_path) |dir| {
+ try zig_args.append("--main-mod-path");
try zig_args.append(dir.getPath(b));
}
lib/std/Build/Cache.zig
@@ -9,6 +9,13 @@ pub const Directory = struct {
path: ?[]const u8,
handle: fs.Dir,
+ pub fn cwd() Directory {
+ return .{
+ .path = null,
+ .handle = fs.cwd(),
+ };
+ }
+
pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
if (self.path) |p| {
// TODO clean way to do this with only 1 allocation
@@ -53,6 +60,10 @@ pub const Directory = struct {
try writer.writeAll(fs.path.sep_str);
}
}
+
+ pub fn eql(self: Directory, other: Directory) bool {
+ return self.handle.fd == other.handle.fd;
+ }
};
gpa: Allocator,
lib/std/Build.zig
@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// Deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -652,7 +655,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile {
.use_llvm = options.use_llvm,
.use_lld = options.use_lld,
.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
- .main_pkg_path = options.main_pkg_path,
+ .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
});
}
@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// Deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -683,7 +689,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile {
.use_llvm = options.use_llvm,
.use_lld = options.use_lld,
.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
- .main_pkg_path = options.main_pkg_path,
+ .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
});
}
@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// Deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile
.use_llvm = options.use_llvm,
.use_lld = options.use_lld,
.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
- .main_pkg_path = options.main_pkg_path,
+ .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
});
}
@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// Deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile
.use_llvm = options.use_llvm,
.use_lld = options.use_lld,
.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
- .main_pkg_path = options.main_pkg_path,
+ .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
});
}
@@ -769,6 +781,9 @@ pub const TestOptions = struct {
use_llvm: ?bool = null,
use_lld: ?bool = null,
zig_lib_dir: ?LazyPath = null,
+ main_mod_path: ?LazyPath = null,
+
+ /// Deprecated; use `main_mod_path`.
main_pkg_path: ?LazyPath = null,
};
@@ -787,7 +802,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile {
.use_llvm = options.use_llvm,
.use_lld = options.use_lld,
.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
- .main_pkg_path = options.main_pkg_path,
+ .main_mod_path = options.main_mod_path orelse options.main_pkg_path,
});
}
src/Package/Fetch.zig
@@ -27,59 +27,84 @@
//! All of this must be done with only referring to the state inside this struct
//! because this work will be done in a dedicated thread.
-/// Try to avoid this as much as possible since arena will have less contention.
-gpa: Allocator,
arena: std.heap.ArenaAllocator,
location: Location,
location_tok: std.zig.Ast.TokenIndex,
hash_tok: std.zig.Ast.TokenIndex,
-global_cache: Cache.Directory,
-parent_package_root: Path,
+parent_package_root: Package.Path,
parent_manifest_ast: ?*const std.zig.Ast,
prog_node: *std.Progress.Node,
-http_client: *std.http.Client,
-thread_pool: *ThreadPool,
job_queue: *JobQueue,
-wait_group: *WaitGroup,
+/// If true, don't add an error for a missing hash. This flag is not passed
+/// down to recursive dependencies. It's intended to be used only be the CLI.
+omit_missing_hash_error: bool,
// Above this are fields provided as inputs to `run`.
// Below this are fields populated by `run`.
/// This will either be relative to `global_cache`, or to the build root of
/// the root package.
-package_root: Path,
+package_root: Package.Path,
error_bundle: std.zig.ErrorBundle.Wip,
manifest: ?Manifest,
-manifest_ast: ?*std.zig.Ast,
-actual_hash: Digest,
+manifest_ast: std.zig.Ast,
+actual_hash: Manifest.Digest,
/// Fetch logic notices whether a package has a build.zig file and sets this flag.
has_build_zig: bool,
/// Indicates whether the task aborted due to an out-of-memory condition.
oom_flag: bool,
+/// Contains shared state among all `Fetch` tasks.
pub const JobQueue = struct {
mutex: std.Thread.Mutex = .{},
-};
-
-pub const Digest = [Manifest.Hash.digest_length]u8;
-pub const MultiHashHexDigest = [hex_multihash_len]u8;
-
-pub const Path = struct {
- root_dir: Cache.Directory,
- /// The path, relative to the root dir, that this `Path` represents.
- /// Empty string means the root_dir is the path.
- sub_path: []const u8 = "",
+ /// Protected by `mutex`.
+ table: Table = .{},
+ /// `table` may be missing some tasks such as ones that failed, so this
+ /// field contains references to all of them.
+ /// Protected by `mutex`.
+ all_fetches: std.ArrayListUnmanaged(*Fetch) = .{},
+
+ http_client: *std.http.Client,
+ thread_pool: *ThreadPool,
+ wait_group: WaitGroup = .{},
+ global_cache: Cache.Directory,
+ recursive: bool,
+ work_around_btrfs_bug: bool,
+
+ pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
+
+ pub fn deinit(jq: *JobQueue) void {
+ if (jq.all_fetches.items.len == 0) return;
+ const gpa = jq.all_fetches.items[0].arena.child_allocator;
+ jq.table.deinit(gpa);
+ // These must be deinitialized in reverse order because subsequent
+ // `Fetch` instances are allocated in prior ones' arenas.
+ // Sorry, I know it's a bit weird, but it slightly simplifies the
+ // critical section.
+ while (jq.all_fetches.popOrNull()) |f| f.deinit();
+ jq.all_fetches.deinit(gpa);
+ jq.* = undefined;
+ }
};
pub const Location = union(enum) {
remote: Remote,
+ /// A directory found inside the parent package.
relative_path: []const u8,
+ /// Recursive Fetch tasks will never use this Location, but it may be
+ /// passed in by the CLI. Indicates the file contents here should be copied
+ /// into the global package cache. It may be a file relative to the cwd or
+ /// absolute, in which case it should be treated exactly like a `file://`
+ /// URL, or a directory, in which case it should be treated as an
+ /// already-unpacked directory (but still needs to be copied into the
+ /// global package cache and have inclusion rules applied).
+ path_or_url: []const u8,
pub const Remote = struct {
url: []const u8,
/// If this is null it means the user omitted the hash field from a dependency.
/// It will be an error but the logic should still fetch and print the discovered hash.
- hash: ?[hex_multihash_len]u8,
+ hash: ?Manifest.MultiHashHexDigest,
};
};
@@ -92,7 +117,11 @@ pub const RunError = error{
pub fn run(f: *Fetch) RunError!void {
const eb = &f.error_bundle;
- const arena = f.arena_allocator.allocator();
+ const arena = f.arena.allocator();
+ const gpa = f.arena.child_allocator;
+ const cache_root = f.job_queue.global_cache;
+
+ try eb.init(gpa);
// Check the global zig package cache to see if the hash already exists. If
// so, load, parse, and validate the build.zig.zon file therein, and skip
@@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void {
);
f.package_root = try f.parent_package_root.join(arena, sub_path);
try loadManifest(f, f.package_root);
+ if (!f.job_queue.recursive) return;
// Package hashes are used as unique identifiers for packages, so
// we still need one for relative paths.
- const hash = h: {
+ const digest = h: {
var hasher = Manifest.Hash.init(.{});
// This hash is a tuple of:
// * whether it relative to the global cache directory or to the root package
// * the relative file path from there to the build root of the package
- hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle)
+ hasher.update(if (f.package_root.root_dir.eql(cache_root))
&package_hash_prefix_cached
else
&package_hash_prefix_project);
hasher.update(f.package_root.sub_path);
break :h hasher.finalResult();
};
- return queueJobsForDeps(f, hash);
+ return queueJobsForDeps(f, Manifest.hexDigest(digest));
},
.remote => |remote| remote,
+ .path_or_url => |path_or_url| {
+ if (fs.cwd().openIterableDir(path_or_url, .{})) |dir| {
+ var resource: Resource = .{ .dir = dir };
+ return runResource(f, path_or_url, &resource, null);
+ } else |dir_err| {
+ const file_err = if (dir_err == error.NotDir) e: {
+ if (fs.cwd().openFile(path_or_url, .{})) |file| {
+ var resource: Resource = .{ .file = file };
+ return runResource(f, path_or_url, &resource, null);
+ } else |err| break :e err;
+ } else dir_err;
+
+ const uri = std.Uri.parse(path_or_url) catch |uri_err| {
+ return f.fail(0, try eb.printString(
+ "'{s}' could not be recognized as a file path ({s}) or an URL ({s})",
+ .{ path_or_url, @errorName(file_err), @errorName(uri_err) },
+ ));
+ };
+ var resource = try f.initResource(uri);
+ return runResource(f, uri.path, &resource, null);
+ }
+ },
};
+
const s = fs.path.sep_str;
if (remote.hash) |expected_hash| {
const pkg_sub_path = "p" ++ s ++ expected_hash;
- if (f.global_cache.handle.access(pkg_sub_path, .{})) |_| {
+ if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
f.package_root = .{
- .root_dir = f.global_cache,
+ .root_dir = cache_root,
.sub_path = pkg_sub_path,
};
try loadManifest(f, f.package_root);
+ if (!f.job_queue.recursive) return;
return queueJobsForDeps(f, expected_hash);
} else |err| switch (err) {
error.FileNotFound => {},
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{
- try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e),
+ try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e),
}),
- .src_loc = .none,
- .notes_len = 0,
});
return error.FetchFailed;
},
@@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void {
const uri = std.Uri.parse(remote.url) catch |err| return f.fail(
f.location_tok,
- "invalid URI: {s}",
- .{@errorName(err)},
+ try eb.printString("invalid URI: {s}", .{@errorName(err)}),
);
+ var resource = try f.initResource(uri);
+ return runResource(f, uri.path, &resource, remote.hash);
+}
+
+pub fn deinit(f: *Fetch) void {
+ f.error_bundle.deinit();
+ f.arena.deinit();
+}
+
+/// Consumes `resource`, even if an error is returned.
+fn runResource(
+ f: *Fetch,
+ uri_path: []const u8,
+ resource: *Resource,
+ remote_hash: ?Manifest.MultiHashHexDigest,
+) RunError!void {
+ defer resource.deinit();
+ const arena = f.arena.allocator();
+ const eb = &f.error_bundle;
+ const s = fs.path.sep_str;
+ const cache_root = f.job_queue.global_cache;
const rand_int = std.crypto.random.int(u64);
const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
+ const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
var tmp_directory: Cache.Directory = .{
- .path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}),
- .handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir,
+ .path = tmp_directory_path,
+ .handle = handle: {
+ const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch |err| {
+ try eb.addRootErrorMessage(.{
+ .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{
+ tmp_directory_path, @errorName(err),
+ }),
+ });
+ return error.FetchFailed;
+ };
+ break :handle dir.dir;
+ },
};
defer tmp_directory.handle.close();
- var resource = try f.initResource(uri);
- defer resource.deinit(); // releases more than memory
-
- try f.unpackResource(&resource, uri.path, tmp_directory);
+ try unpackResource(f, resource, uri_path, tmp_directory);
// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
// for the file to be missing, in which case this fetched package is
@@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void {
// Compute the package hash based on the remaining files in the temporary
// directory.
- if (builtin.os.tag == .linux and f.work_around_btrfs_bug) {
+ if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) {
// https://github.com/ziglang/zig/issues/17095
tmp_directory.handle.close();
- const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
+ const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
@panic("btrfs workaround failed");
tmp_directory.handle = iterable_dir.dir;
}
- f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter);
+ f.actual_hash = try computeHash(f, tmp_directory, filter);
// Rename the temporary directory into the global zig package cache
// directory. If the hash already exists, delete the temporary directory
@@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void {
// package with the different hash is used in the future.
const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash);
- try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path);
+ renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch |err| {
+ const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
+ const dest = try cache_root.join(arena, &.{dest_pkg_sub_path});
+ try eb.addRootErrorMessage(.{ .msg = try eb.printString(
+ "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",
+ .{ src, dest, @errorName(err) },
+ ) });
+ return error.FetchFailed;
+ };
// Validate the computed hash against the expected hash. If invalid, this
// job is done.
const actual_hex = Manifest.hexDigest(f.actual_hash);
- if (remote.hash) |declared_hash| {
- if (!std.mem.eql(u8, declared_hash, &actual_hex)) {
- return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{
- declared_hash, actual_hex,
- });
+ if (remote_hash) |declared_hash| {
+ if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
+ return f.fail(f.hash_tok, try eb.printString(
+ "hash mismatch: manifest declares {s} but the fetched package has {s}",
+ .{ declared_hash, actual_hex },
+ ));
}
- } else {
+ } else if (!f.omit_missing_hash_error) {
const notes_len = 1;
- try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field");
+ try eb.addRootErrorMessage(.{
+ .msg = try eb.addString("dependency is missing hash field"),
+ .src_loc = try f.srcLoc(f.location_tok),
+ .notes_len = notes_len,
+ });
const notes_start = try eb.reserveNotes(notes_len);
eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
}));
- return error.PackageFetchFailed;
+ return error.FetchFailed;
}
// Spawn a new fetch job for each dependency in the manifest file. Use
// a mutex and a hash map so that redundant jobs do not get queued up.
- return queueJobsForDeps(f, .{ .hash = f.actual_hash });
+ if (!f.job_queue.recursive) return;
+ return queueJobsForDeps(f, actual_hex);
}
/// This function populates `f.manifest` or leaves it `null`.
-fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
+fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {
const eb = &f.error_bundle;
- const arena = f.arena_allocator.allocator();
- const manifest_bytes = pkg_root.readFileAllocOptions(
+ const arena = f.arena.allocator();
+ const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
arena,
- Manifest.basename,
+ try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
Manifest.max_bytes,
null,
1,
@@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
) catch |err| switch (err) {
error.FileNotFound => return,
else => |e| {
- const file_path = try pkg_root.join(arena, .{Manifest.basename});
+ const file_path = try pkg_root.join(arena, Manifest.basename);
try eb.addRootErrorMessage(.{
- .msg = try eb.printString("unable to load package manifest '{s}': {s}", .{
+ .msg = try eb.printString("unable to load package manifest '{}': {s}", .{
file_path, @errorName(e),
}),
- .src_loc = .none,
- .notes_len = 0,
});
+ return error.FetchFailed;
},
};
- var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
- f.manifest_ast = ast;
+ const ast = &f.manifest_ast;
+ ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
if (ast.errors.len > 0) {
- const file_path = try pkg_root.join(arena, .{Manifest.basename});
- try main.putAstErrorsIntoBundle(arena, ast, file_path, eb);
- return error.PackageFetchFailed;
+ const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root});
+ try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
+ return error.FetchFailed;
}
- f.manifest = try Manifest.parse(arena, ast);
+ f.manifest = try Manifest.parse(arena, ast.*);
+ const manifest = &f.manifest.?;
- if (f.manifest.errors.len > 0) {
- const file_path = try pkg_root.join(arena, .{Manifest.basename});
+ if (manifest.errors.len > 0) {
+ const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
const token_starts = ast.tokens.items(.start);
- for (f.manifest.errors) |msg| {
+ for (manifest.errors) |msg| {
const start_loc = ast.tokenLocation(0, msg.tok);
try eb.addRootErrorMessage(.{
.msg = try eb.addString(msg.msg),
.src_loc = try eb.addSourceLocation(.{
- .src_path = try eb.addString(file_path),
+ .src_path = src_path,
.span_start = token_starts[msg.tok],
.span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len),
.span_main = token_starts[msg.tok] + msg.off,
@@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
.column = @intCast(start_loc.column),
.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
}),
- .notes_len = 0,
});
}
- return error.PackageFetchFailed;
+ return error.FetchFailed;
}
}
-fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
+fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void {
+ assert(f.job_queue.recursive);
+
// If the package does not have a build.zig.zon file then there are no dependencies.
const manifest = f.manifest orelse return;
const new_fetches = nf: {
+ const deps = manifest.dependencies.values();
+ const gpa = f.arena.child_allocator;
// Grab the new tasks into a temporary buffer so we can unlock that mutex
// as fast as possible.
// This overallocates any fetches that get skipped by the `continue` in the
// loop below.
- const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count());
+ const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len);
var new_fetch_index: usize = 0;
- f.job_queue.lock();
- defer f.job_queue.unlock();
+ f.job_queue.mutex.lock();
+ defer f.job_queue.mutex.unlock();
+
+ try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
+ try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1));
// It is impossible for there to be a collision here. Consider all three cases:
// * Correct hash is provided by manifest.
// - Redundant jobs are skipped in the loop below.
- // * Incorrect has is provided by manifest.
+ // * Incorrect hash is provided by manifest.
// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
// * Hash is not provided by manifest.
// - Hash missing error emitted; `queueJobsForDeps` is not called.
- try f.job_queue.finish(hash, f, new_fetches.len);
+ f.job_queue.table.putAssumeCapacityNoClobber(hash, f);
- for (manifest.dependencies.values()) |dep| {
+ for (deps) |dep| {
+ const new_fetch = &new_fetches[new_fetch_index];
const location: Location = switch (dep.location) {
.url => |url| .{ .remote = .{
.url = url,
- .hash = if (dep.hash) |h| h[0..hex_multihash_len].* else null,
+ .hash = h: {
+ const h = dep.hash orelse break :h null;
+ const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len;
+ const multihash_digest = h[0..digest_len].*;
+ const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
+ if (gop.found_existing) continue;
+ gop.value_ptr.* = new_fetch;
+ break :h multihash_digest;
+ },
} },
.path => |path| .{ .relative_path = path },
};
- const new_fetch = &new_fetches[new_fetch_index];
- const already_done = f.job_queue.add(location, new_fetch);
- if (already_done) continue;
new_fetch_index += 1;
-
+ f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
new_fetch.* = .{
- .gpa = f.gpa,
- .arena = std.heap.ArenaAllocator.init(f.gpa),
+ .arena = std.heap.ArenaAllocator.init(gpa),
.location = location,
.location_tok = dep.location_tok,
.hash_tok = dep.hash_tok,
- .global_cache = f.global_cache,
.parent_package_root = f.package_root,
- .parent_manifest_ast = f.manifest_ast.?,
+ .parent_manifest_ast = &f.manifest_ast,
.prog_node = f.prog_node,
- .http_client = f.http_client,
- .thread_pool = f.thread_pool,
.job_queue = f.job_queue,
- .wait_group = f.wait_group,
+ .omit_missing_hash_error = false,
.package_root = undefined,
- .error_bundle = .{},
+ .error_bundle = undefined,
.manifest = null,
- .manifest_ast = null,
+ .manifest_ast = undefined,
.actual_hash = undefined,
.has_build_zig = false,
+ .oom_flag = false,
};
}
@@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
};
// Now it's time to give tasks to the thread pool.
- for (new_fetches) |new_fetch| {
- f.wait_group.start();
- f.thread_pool.spawn(workerRun, .{f}) catch |err| switch (err) {
+ const thread_pool = f.job_queue.thread_pool;
+
+ for (new_fetches) |*new_fetch| {
+ f.job_queue.wait_group.start();
+ thread_pool.spawn(workerRun, .{new_fetch}) catch |err| switch (err) {
error.OutOfMemory => {
new_fetch.oom_flag = true;
- f.wait_group.finish();
+ f.job_queue.wait_group.finish();
continue;
},
};
@@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
}
fn workerRun(f: *Fetch) void {
- defer f.wait_group.finish();
+ defer f.job_queue.wait_group.finish();
run(f) catch |err| switch (err) {
error.OutOfMemory => f.oom_flag = true,
- error.FetchFailed => {}, // See `error_bundle`.
+ error.FetchFailed => {
+ // Nothing to do because the errors are already reported in `error_bundle`,
+ // and a reference is kept to the `Fetch` task inside `all_fetches`.
+ },
};
}
-fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void {
- const ast = f.parent_manifest_ast;
- const token_starts = ast.tokens.items(.start);
- const start_loc = ast.tokenLocation(0, msg_tok);
+fn srcLoc(
+ f: *Fetch,
+ tok: std.zig.Ast.TokenIndex,
+) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex {
+ const ast = f.parent_manifest_ast orelse return .none;
const eb = &f.error_bundle;
- const file_path = try f.parent_package_root.join(f.arena, Manifest.basename);
+ const token_starts = ast.tokens.items(.start);
+ const start_loc = ast.tokenLocation(0, tok);
+ const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root});
const msg_off = 0;
+ return eb.addSourceLocation(.{
+ .src_path = src_path,
+ .span_start = token_starts[tok],
+ .span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len),
+ .span_main = token_starts[tok] + msg_off,
+ .line = @intCast(start_loc.line),
+ .column = @intCast(start_loc.column),
+ .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
+ });
+}
+fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
+ const eb = &f.error_bundle;
try eb.addRootErrorMessage(.{
.msg = msg_str,
- .src_loc = try eb.addSourceLocation(.{
- .src_path = try eb.addString(file_path),
- .span_start = token_starts[msg_tok],
- .span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len),
- .span_main = token_starts[msg_tok] + msg_off,
- .line = @intCast(start_loc.line),
- .column = @intCast(start_loc.column),
- .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
- }),
- .notes_len = 0,
+ .src_loc = try f.srcLoc(msg_tok),
});
-
return error.FetchFailed;
}
const Resource = union(enum) {
file: fs.File,
http_request: std.http.Client.Request,
- git_fetch_stream: git.Session.FetchStream,
+ git: Git,
dir: fs.IterableDir,
+
+ const Git = struct {
+ fetch_stream: git.Session.FetchStream,
+ want_oid: [git.oid_length]u8,
+ };
+
+ fn deinit(resource: *Resource) void {
+ switch (resource.*) {
+ .file => |*file| file.close(),
+ .http_request => |*req| req.deinit(),
+ .git => |*git_resource| git_resource.fetch_stream.deinit(),
+ .dir => |*dir| dir.close(),
+ }
+ resource.* = undefined;
+ }
+
+ fn reader(resource: *Resource) std.io.AnyReader {
+ return .{
+ .context = resource,
+ .readFn = read,
+ };
+ }
+
+ fn read(context: *const anyopaque, buffer: []u8) anyerror!usize {
+ const resource: *Resource = @constCast(@ptrCast(@alignCast(context)));
+ switch (resource.*) {
+ .file => |*f| return f.read(buffer),
+ .http_request => |*r| return r.read(buffer),
+ .git => |*g| return g.fetch_stream.read(buffer),
+ .dir => unreachable,
+ }
+ }
};
const FileType = enum {
@@ -468,30 +613,52 @@ const FileType = enum {
};
fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
- const gpa = f.gpa;
- const arena = f.arena_allocator.allocator();
+ const gpa = f.arena.child_allocator;
+ const arena = f.arena.allocator();
const eb = &f.error_bundle;
if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{
- .file = try f.parent_package_root.openFile(uri.path, .{}),
+ .file = f.parent_package_root.openFile(uri.path, .{}) catch |err| {
+ return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
+ f.parent_package_root, uri.path, @errorName(err),
+ }));
+ },
};
+ const http_client = f.job_queue.http_client;
+
if (ascii.eqlIgnoreCase(uri.scheme, "http") or
ascii.eqlIgnoreCase(uri.scheme, "https"))
{
var h = std.http.Headers{ .allocator = gpa };
defer h.deinit();
- var req = try f.http_client.request(.GET, uri, h, .{});
+ var req = http_client.request(.GET, uri, h, .{}) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to connect to server: {s}",
+ .{@errorName(err)},
+ ));
+ };
errdefer req.deinit(); // releases more than memory
- try req.start(.{});
- try req.wait();
+ req.start(.{}) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "HTTP request failed: {s}",
+ .{@errorName(err)},
+ ));
+ };
+ req.wait() catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "invalid HTTP response: {s}",
+ .{@errorName(err)},
+ ));
+ };
if (req.response.status != .ok) {
- return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{
- @intFromEnum(req.response.status), req.response.status.phrase() orelse "",
- });
+ return f.fail(f.location_tok, try eb.printString(
+ "bad HTTP response code: '{d} {s}'",
+ .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" },
+ ));
}
return .{ .http_request = req };
@@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
var transport_uri = uri;
transport_uri.scheme = uri.scheme["git+".len..];
var redirect_uri: []u8 = undefined;
- var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri };
- session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) {
+ var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
+ session.discoverCapabilities(gpa, &redirect_uri) catch |err| switch (err) {
error.Redirected => {
defer gpa.free(redirect_uri);
- return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri});
+ return f.fail(f.location_tok, try eb.printString(
+ "repository moved to {s}",
+ .{redirect_uri},
+ ));
+ },
+ else => |e| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to discover remote git server capabilities: {s}",
+ .{@errorName(e)},
+ ));
},
- else => |other| return other,
};
const want_oid = want_oid: {
@@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
- var ref_iterator = try session.listRefs(gpa, .{
+ var ref_iterator = session.listRefs(gpa, .{
.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
.include_peeled = true,
- });
+ }) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to list refs: {s}",
+ .{@errorName(err)},
+ ));
+ };
defer ref_iterator.deinit();
- while (try ref_iterator.next()) |ref| {
+ while (ref_iterator.next() catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to iterate refs: {s}",
+ .{@errorName(err)},
+ ));
+ }) |ref| {
if (std.mem.eql(u8, ref.name, want_ref) or
std.mem.eql(u8, ref.name, want_ref_head) or
std.mem.eql(u8, ref.name, want_ref_tag))
@@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
break :want_oid ref.peeled orelse ref.oid;
}
}
- return f.fail(f.location_tok, "ref not found: {s}", .{want_ref});
+ return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
};
if (uri.fragment == null) {
const notes_len = 1;
- try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref");
+ try eb.addRootErrorMessage(.{
+ .msg = try eb.addString("url field is missing an explicit ref"),
+ .src_loc = try f.srcLoc(f.location_tok),
+ .notes_len = notes_len,
+ });
const notes_start = try eb.reserveNotes(notes_len);
eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
.msg = try eb.printString("try .url = \"{+/}#{}\",", .{
uri, std.fmt.fmtSliceHexLower(&want_oid),
}),
}));
- return error.PackageFetchFailed;
+ return error.FetchFailed;
}
var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
std.fmt.fmtSliceHexLower(&want_oid),
}) catch unreachable;
- var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});
+ var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to create fetch stream: {s}",
+ .{@errorName(err)},
+ ));
+ };
errdefer fetch_stream.deinit();
- return .{ .git_fetch_stream = fetch_stream };
+ return .{ .git = .{
+ .fetch_stream = fetch_stream,
+ .want_oid = want_oid,
+ } };
}
- return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
+ return f.fail(f.location_tok, try eb.printString(
+ "unsupported URL scheme: {s}",
+ .{uri.scheme},
+ ));
}
fn unpackResource(
@@ -565,52 +765,62 @@ fn unpackResource(
uri_path: []const u8,
tmp_directory: Cache.Directory,
) RunError!void {
+ const eb = &f.error_bundle;
const file_type = switch (resource.*) {
.file => FileType.fromPath(uri_path) orelse
- return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}),
+ return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
.http_request => |req| ft: {
// Content-Type takes first precedence.
const content_type = req.response.headers.getFirstValue("Content-Type") orelse
- return f.fail(f.location_tok, "missing 'Content-Type' header", .{});
+ return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
if (ascii.eqlIgnoreCase(content_type, "application/x-tar"))
- return .tar;
+ break :ft .tar;
if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
{
- return .@"tar.gz";
+ break :ft .@"tar.gz";
}
if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))
- return .@"tar.xz";
+ break :ft .@"tar.xz";
if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
- return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{
- content_type,
- });
+ return f.fail(f.location_tok, try eb.printString(
+ "unrecognized 'Content-Type' header: '{s}'",
+ .{content_type},
+ ));
}
// Next, the filename from 'content-disposition: attachment' takes precedence.
if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| {
- break :ft FileType.fromContentDisposition(cd_header) orelse
- return f.fail(
- f.location_tok,
- "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
- .{cd_header},
- );
+ break :ft FileType.fromContentDisposition(cd_header) orelse {
+ return f.fail(f.location_tok, try eb.printString(
+ "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
+ .{cd_header},
+ ));
+ };
}
// Finally, the path from the URI is used.
- break :ft FileType.fromPath(uri_path) orelse
- return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path});
+ break :ft FileType.fromPath(uri_path) orelse {
+ return f.fail(f.location_tok, try eb.printString(
+ "unknown file type: '{s}'",
+ .{uri_path},
+ ));
+ };
},
- .git_fetch_stream => return .git_pack,
- .dir => |dir| {
- try f.recursiveDirectoryCopy(dir, tmp_directory.handle);
- return;
+
+ .git => .git_pack,
+
+ .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to copy directory '{s}': {s}",
+ .{ uri_path, @errorName(err) },
+ ));
},
};
@@ -618,7 +828,14 @@ fn unpackResource(
.tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
.@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
.@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
- .git_pack => try unpackGitPack(f, tmp_directory.handle, resource),
+ .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
+ error.FetchFailed => return error.FetchFailed,
+ error.OutOfMemory => return error.OutOfMemory,
+ else => |e| return f.fail(f.location_tok, try eb.printString(
+ "unable to unpack git files: {s}",
+ .{@errorName(e)},
+ )),
+ },
}
}
@@ -628,11 +845,17 @@ fn unpackTarballCompressed(
resource: *Resource,
comptime Compression: type,
) RunError!void {
- const gpa = f.gpa;
+ const gpa = f.arena.child_allocator;
+ const eb = &f.error_bundle;
const reader = resource.reader();
var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
- var decompress = try Compression.decompress(gpa, br.reader());
+ var decompress = Compression.decompress(gpa, br.reader()) catch |err| {
+ return f.fail(f.location_tok, try eb.printString(
+ "unable to decompress tarball: {s}",
+ .{@errorName(err)},
+ ));
+ };
defer decompress.deinit();
return unpackTarball(f, out_dir, decompress.reader());
@@ -640,11 +863,12 @@ fn unpackTarballCompressed(
fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
const eb = &f.error_bundle;
+ const gpa = f.arena.child_allocator;
- var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa };
+ var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
defer diagnostics.deinit();
- try std.tar.pipeToFileSystem(out_dir, reader, .{
+ std.tar.pipeToFileSystem(out_dir, reader, .{
.diagnostics = &diagnostics,
.strip_components = 1,
// TODO: we would like to set this to executable_bit_only, but two
@@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
// 2. the hashing algorithm here needs to support detecting the is_executable
// bit on Windows from the ACLs (see the isExecutable function).
.mode_mode = .ignore,
- .filter = .{ .exclude_empty_directories = true },
- });
+ .exclude_empty_directories = true,
+ }) catch |err| return f.fail(f.location_tok, try eb.printString(
+ "unable to unpack tarball to temporary directory: {s}",
+ .{@errorName(err)},
+ ));
if (diagnostics.errors.items.len > 0) {
const notes_len: u32 = @intCast(diagnostics.errors.items.len);
- try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball");
+ try eb.addRootErrorMessage(.{
+ .msg = try eb.addString("unable to unpack tarball"),
+ .src_loc = try f.srcLoc(f.location_tok),
+ .notes_len = notes_len,
+ });
const notes_start = try eb.reserveNotes(notes_len);
for (diagnostics.errors.items, notes_start..) |item, note_i| {
switch (item) {
@@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
},
}
}
- return error.InvalidTarball;
+ return error.FetchFailed;
}
}
-fn unpackGitPack(
- f: *Fetch,
- out_dir: fs.Dir,
- resource: *Resource,
- want_oid: git.Oid,
-) !void {
+fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void {
const eb = &f.error_bundle;
- const gpa = f.gpa;
- const reader = resource.reader();
+ const gpa = f.arena.child_allocator;
+ const want_oid = resource.git.want_oid;
+ const reader = resource.git.fetch_stream.reader();
// The .git directory is used to store the packfile and associated index, but
// we do not attempt to replicate the exact structure of a real .git
// directory, since that isn't relevant for fetching a package.
@@ -700,13 +927,13 @@ fn unpackGitPack(
var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
defer pack_file.close();
var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
- try fifo.pump(reader.reader(), pack_file.writer());
+ try fifo.pump(reader, pack_file.writer());
try pack_file.sync();
var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
defer index_file.close();
{
- var index_prog_node = reader.prog_node.start("Index pack", 0);
+ var index_prog_node = f.prog_node.start("Index pack", 0);
defer index_prog_node.end();
index_prog_node.activate();
var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
@@ -716,7 +943,7 @@ fn unpackGitPack(
}
{
- var checkout_prog_node = reader.prog_node.start("Checkout", 0);
+ var checkout_prog_node = f.prog_node.start("Checkout", 0);
defer checkout_prog_node.end();
checkout_prog_node.activate();
var repository = try git.Repository.init(gpa, pack_file, index_file);
@@ -727,7 +954,11 @@ fn unpackGitPack(
if (diagnostics.errors.items.len > 0) {
const notes_len: u32 = @intCast(diagnostics.errors.items.len);
- try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile");
+ try eb.addRootErrorMessage(.{
+ .msg = try eb.addString("unable to unpack packfile"),
+ .src_loc = try f.srcLoc(f.location_tok),
+ .notes_len = notes_len,
+ });
const notes_start = try eb.reserveNotes(notes_len);
for (diagnostics.errors.items, notes_start..) |item, note_i| {
switch (item) {
@@ -748,9 +979,10 @@ fn unpackGitPack(
try out_dir.deleteTree(".git");
}
-fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void {
+fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void {
+ const gpa = f.arena.child_allocator;
// Recursive directory copy.
- var it = try dir.walk(f.gpa);
+ var it = try dir.walk(gpa);
defer it.deinit();
while (try it.next()) |entry| {
switch (entry.kind) {
@@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache(
/// the hash are not present on the file system. Empty directories are *not
/// hashed* and must not be present on the file system when calling this
/// function.
-fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest {
+fn computeHash(
+ f: *Fetch,
+ tmp_directory: Cache.Directory,
+ filter: Filter,
+) RunError!Manifest.Digest {
// All the path name strings need to be in memory for sorting.
- const arena = f.arena_allocator.allocator();
- const gpa = f.gpa;
+ const arena = f.arena.allocator();
+ const gpa = f.arena.child_allocator;
+ const eb = &f.error_bundle;
+ const thread_pool = f.job_queue.thread_pool;
// Collect all files, recursively, then sort.
var all_files = std.ArrayList(*HashedFile).init(gpa);
defer all_files.deinit();
- var walker = try pkg_dir.walk(gpa);
+ var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa);
defer walker.deinit();
{
@@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
var wait_group: WaitGroup = .{};
// `computeHash` is called from a worker thread so there must not be
// any waiting without working or a deadlock could occur.
- defer wait_group.waitAndWork();
-
- while (try walker.next()) |entry| {
+ defer thread_pool.waitAndWork(&wait_group);
+
+ while (walker.next() catch |err| {
+ try eb.addRootErrorMessage(.{ .msg = try eb.printString(
+ "unable to walk temporary directory '{}': {s}",
+ .{ tmp_directory, @errorName(err) },
+ ) });
+ return error.FetchFailed;
+ }) |entry| {
_ = filter; // TODO: apply filter rules here
const kind: HashedFile.Kind = switch (entry.kind) {
.directory => continue,
.file => .file,
.sym_link => .sym_link,
- else => return error.IllegalFileTypeInPackage,
+ else => return f.fail(f.location_tok, try eb.printString(
+ "package contains '{s}' which has illegal file type '{s}'",
+ .{ entry.path, @tagName(entry.kind) },
+ )),
};
- if (std.mem.eql(u8, entry.path, build_zig_basename))
+ if (std.mem.eql(u8, entry.path, Package.build_zig_basename))
f.has_build_zig = true;
const hashed_file = try arena.create(HashedFile);
@@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
.failure = undefined, // to be populated by the worker
};
wait_group.start();
- try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
+ try thread_pool.spawn(workerHashFile, .{
+ tmp_directory.handle, hashed_file, &wait_group,
+ });
try all_files.append(hashed_file);
}
@@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
var hasher = Manifest.Hash.init(.{});
var any_failures = false;
- const eb = &f.error_bundle;
for (all_files.items) |hashed_file| {
hashed_file.failure catch |err| {
any_failures = true;
try eb.addRootErrorMessage(.{
- .msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}),
- .src_loc = try eb.addSourceLocation(.{
- .src_path = try eb.addString(hashed_file.fs_path),
- .span_start = 0,
- .span_end = 0,
- .span_main = 0,
+ .msg = try eb.printString("unable to hash '{s}': {s}", .{
+ hashed_file.fs_path, @errorName(err),
}),
- .notes_len = 0,
});
};
hasher.update(&hashed_file.hash);
@@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool {
const HashedFile = struct {
fs_path: []const u8,
normalized_path: []const u8,
- hash: Digest,
+ hash: Manifest.Digest,
failure: Error!void,
kind: Kind,
@@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
return normalized;
}
-pub const Filter = struct {
+const Filter = struct {
include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
/// sub_path is relative to the tarball root.
@@ -990,12 +1233,9 @@ pub const Filter = struct {
}
};
-const build_zig_basename = @import("../Package.zig").build_zig_basename;
-const hex_multihash_len = 2 * Manifest.multihash_len;
-
// These are random bytes.
-const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
-const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
+const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
+const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
const builtin = @import("builtin");
const std = @import("std");
@@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig");
const Fetch = @This();
const main = @import("../main.zig");
const git = @import("../git.zig");
+const Package = @import("../Package.zig");
src/Package/Module.zig
@@ -0,0 +1,32 @@
+//! Corresponds to something that Zig source code can `@import`.
+//! Not to be confused with src/Module.zig which should be renamed
+//! to something else. https://github.com/ziglang/zig/issues/14307
+
+/// Only files inside this directory can be imported.
+root: Package.Path,
+/// Relative to `root`. May contain path separators.
+root_src_path: []const u8,
+/// The dependency table of this module. Shared dependencies such as 'std',
+/// 'builtin', and 'root' are not specified in every dependency table, but
+/// instead only in the table of `main_pkg`. `Module.importFile` is
+/// responsible for detecting these names and using the correct package.
+deps: Deps = .{},
+
+pub const Deps = std.StringHashMapUnmanaged(*Module);
+
+pub const Tree = struct {
+ /// Each `Package` exposes a `Module` with build.zig as its root source file.
+ build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module),
+};
+
+pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module {
+ const new = try allocator.create(Module);
+ new.* = m;
+ return new;
+}
+
+const Module = @This();
+const Package = @import("../Package.zig");
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;
src/Compilation.zig
@@ -273,8 +273,8 @@ const Job = union(enum) {
/// The source file containing the Decl has been updated, and so the
/// Decl may need its line number information updated in the debug info.
update_line_number: Module.Decl.Index,
- /// The main source file for the package needs to be analyzed.
- analyze_pkg: *Package,
+ /// The main source file for the module needs to be analyzed.
+ analyze_mod: *Package.Module,
/// one of the glibc static objects
glibc_crt_file: glibc.CRTFile,
@@ -414,7 +414,7 @@ pub const MiscTask = enum {
compiler_rt,
libssp,
zig_libc,
- analyze_pkg,
+ analyze_mod,
@"musl crti.o",
@"musl crtn.o",
@@ -544,7 +544,7 @@ pub const InitOptions = struct {
global_cache_directory: Directory,
target: Target,
root_name: []const u8,
- main_pkg: ?*Package,
+ main_mod: ?*Package.Module,
output_mode: std.builtin.OutputMode,
thread_pool: *ThreadPool,
dynamic_linker: ?[]const u8 = null,
@@ -736,53 +736,53 @@ pub const InitOptions = struct {
pdb_out_path: ?[]const u8 = null,
};
-fn addPackageTableToCacheHash(
+fn addModuleTableToCacheHash(
hash: *Cache.HashHelper,
arena: *std.heap.ArenaAllocator,
- pkg_table: Package.Table,
- seen_table: *std.AutoHashMap(*Package, void),
+ mod_table: Package.Module.Deps,
+ seen_table: *std.AutoHashMap(*Package.Module, void),
hash_type: union(enum) { path_bytes, files: *Cache.Manifest },
) (error{OutOfMemory} || std.os.GetCwdError)!void {
const allocator = arena.allocator();
- const packages = try allocator.alloc(Package.Table.KV, pkg_table.count());
+ const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count());
{
// Copy over the hashmap entries to our slice
- var table_it = pkg_table.iterator();
+ var table_it = mod_table.iterator();
var idx: usize = 0;
while (table_it.next()) |entry| : (idx += 1) {
- packages[idx] = .{
+ modules[idx] = .{
.key = entry.key_ptr.*,
.value = entry.value_ptr.*,
};
}
}
// Sort the slice by package name
- mem.sort(Package.Table.KV, packages, {}, struct {
- fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool {
+ mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct {
+ fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool {
return std.mem.lessThan(u8, lhs.key, rhs.key);
}
}.lessThan);
- for (packages) |pkg| {
- if ((try seen_table.getOrPut(pkg.value)).found_existing) continue;
+ for (modules) |mod| {
+ if ((try seen_table.getOrPut(mod.value)).found_existing) continue;
// Finally insert the package name and path to the cache hash.
- hash.addBytes(pkg.key);
+ hash.addBytes(mod.key);
switch (hash_type) {
.path_bytes => {
- hash.addBytes(pkg.value.root_src_path);
- hash.addOptionalBytes(pkg.value.root_src_directory.path);
+ hash.addBytes(mod.value.root_src_path);
+ hash.addOptionalBytes(mod.value.root_src_directory.path);
},
.files => |man| {
- const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{
- pkg.value.root_src_path,
+ const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{
+ mod.value.root_src_path,
});
_ = try man.addFile(pkg_zig_file, null);
},
}
- // Recurse to handle the package's dependencies
- try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type);
+ // Recurse to handle the module's dependencies
+ try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type);
}
}
@@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
break :blk true;
// If we have no zig code to compile, no need for LLVM.
- if (options.main_pkg == null)
+ if (options.main_mod == null)
break :blk false;
// If LLVM does not support the target, then we can't use it.
@@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
// compiler state, the second clause here can be removed so that incremental
// cache mode is used for LLVM backend too. We need some fuzz testing before
// that can be enabled.
- const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching)
+ const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching)
CacheMode.whole
else
options.cache_mode;
@@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
if (use_llvm) {
// If stage1 generates an object file, self-hosted linker is not
// yet sophisticated enough to handle that.
- break :blk options.main_pkg != null;
+ break :blk options.main_mod != null;
}
break :blk false;
@@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
// TODO audit this and make sure everything is in it
- const module: ?*Module = if (options.main_pkg) |main_pkg| blk: {
+ const module: ?*Module = if (options.main_mod) |main_mod| blk: {
// Options that are specific to zig source files, that cannot be
// modified between incremental updates.
var hash = cache.hash;
@@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
// do want to namespace different source file names because they are
// likely different compilations and therefore this would be likely to
// cause cache hits.
- hash.addBytes(main_pkg.root_src_path);
- hash.addOptionalBytes(main_pkg.root_src_directory.path);
+ hash.addBytes(main_mod.root_src_path);
+ hash.addOptionalBytes(main_mod.root.root_dir.path);
+ hash.addBytes(main_mod.root.sub_path);
{
- var seen_table = std.AutoHashMap(*Package, void).init(arena);
- try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
+ var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
+ try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes);
}
},
.whole => {
@@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
.path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
};
- const builtin_pkg = try Package.createWithDir(
- gpa,
- zig_cache_artifact_directory,
- null,
- "builtin.zig",
- );
- errdefer builtin_pkg.destroy(gpa);
+ const builtin_mod = try Package.Module.create(arena, .{
+ .root = .{ .root_dir = zig_cache_artifact_directory },
+ .root_src_path = "builtin.zig",
+ });
- // When you're testing std, the main module is std. In that case, we'll just set the std
- // module to the main one, since avoiding the errors caused by duplicating it is more
- // effort than it's worth.
- const main_pkg_is_std = m: {
+ // When you're testing std, the main module is std. In that case,
+ // we'll just set the std module to the main one, since avoiding
+ // the errors caused by duplicating it is more effort than it's
+ // worth.
+ const main_mod_is_std = m: {
const std_path = try std.fs.path.resolve(arena, &[_][]const u8{
options.zig_lib_directory.path orelse ".",
"std",
"std.zig",
});
- defer arena.free(std_path);
const main_path = try std.fs.path.resolve(arena, &[_][]const u8{
- main_pkg.root_src_directory.path orelse ".",
- main_pkg.root_src_path,
+ main_mod.root.root_dir.path orelse ".",
+ main_mod.root.sub_path,
+ main_mod.root_src_path,
});
- defer arena.free(main_path);
break :m mem.eql(u8, main_path, std_path);
};
- const std_pkg = if (main_pkg_is_std)
- main_pkg
+ const std_mod = if (main_mod_is_std)
+ main_mod
else
try Package.createWithDir(
gpa,
@@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
"std.zig",
);
- errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa);
+ errdefer if (!main_mod_is_std) std_mod.destroy(gpa);
- const root_pkg = if (options.is_test) root_pkg: {
+ const root_mod = if (options.is_test) root_mod: {
const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: {
const test_dir = std.fs.path.dirname(test_runner);
const basename = std.fs.path.basename(test_runner);
const pkg = try Package.create(gpa, test_dir, basename);
- // copy package table from main_pkg to root_pkg
- pkg.table = try main_pkg.table.clone(gpa);
+ // copy module table from main_mod to root_mod
+ pkg.deps = try main_mod.deps.clone(gpa);
break :test_pkg pkg;
} else try Package.createWithDir(
gpa,
@@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
);
errdefer test_pkg.destroy(gpa);
- break :root_pkg test_pkg;
- } else main_pkg;
- errdefer if (options.is_test) root_pkg.destroy(gpa);
+ break :root_mod test_pkg;
+ } else main_mod;
+ errdefer if (options.is_test) root_mod.destroy(gpa);
- const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: {
- break :compiler_rt_pkg try Package.createWithDir(
+ const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: {
+ break :compiler_rt_mod try Package.createWithDir(
gpa,
options.zig_lib_directory,
null,
"compiler_rt.zig",
);
} else null;
- errdefer if (compiler_rt_pkg) |p| p.destroy(gpa);
+ errdefer if (compiler_rt_mod) |p| p.destroy(gpa);
- try main_pkg.add(gpa, "builtin", builtin_pkg);
- try main_pkg.add(gpa, "root", root_pkg);
- try main_pkg.add(gpa, "std", std_pkg);
+ try main_mod.add(gpa, "builtin", builtin_mod);
+ try main_mod.add(gpa, "root", root_mod);
+ try main_mod.add(gpa, "std", std_mod);
- if (compiler_rt_pkg) |p| {
- try main_pkg.add(gpa, "compiler_rt", p);
+ if (compiler_rt_mod) |p| {
+ try main_mod.add(gpa, "compiler_rt", p);
}
// Pre-open the directory handles for cached ZIR code so that it does not need
@@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
module.* = .{
.gpa = gpa,
.comp = comp,
- .main_pkg = main_pkg,
- .root_pkg = root_pkg,
+ .main_mod = main_mod,
+ .root_mod = root_mod,
.zig_cache_artifact_directory = zig_cache_artifact_directory,
.global_zir_cache = global_zir_cache,
.local_zir_cache = local_zir_cache,
@@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director
// This is only for cleanup purposes; Module.deinit calls close
// on the handle of zig_cache_artifact_directory.
if (comp.bin_file.options.module) |module| {
- const builtin_pkg = module.main_pkg.table.get("builtin").?;
- module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
+ const builtin_mod = module.main_mod.deps.get("builtin").?;
+ module.zig_cache_artifact_directory = builtin_mod.root_src_directory;
}
}
@@ -2148,8 +2146,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
// Make sure std.zig is inside the import_table. We unconditionally need
// it for start.zig.
- const std_pkg = module.main_pkg.table.get("std").?;
- _ = try module.importPkg(std_pkg);
+ const std_mod = module.main_mod.deps.get("std").?;
+ _ = try module.importPkg(std_mod);
// Normally we rely on importing std to in turn import the root source file
// in the start code, but when using the stage1 backend that won't happen,
@@ -2158,11 +2156,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
// Likewise, in the case of `zig test`, the test runner is the root source file,
// and so there is nothing to import the main file.
if (comp.bin_file.options.is_test) {
- _ = try module.importPkg(module.main_pkg);
+ _ = try module.importPkg(module.main_mod);
}
- if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
- _ = try module.importPkg(compiler_rt_pkg);
+ if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
+ _ = try module.importPkg(compiler_rt_mod);
}
// Put a work item in for every known source file to detect if
@@ -2185,13 +2183,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
}
}
- try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
+ try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
if (comp.bin_file.options.is_test) {
- try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
+ try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod });
}
- if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
- try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg });
+ if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
+ try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod });
}
}
@@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
comptime assert(link_hash_implementation_version == 10);
if (comp.bin_file.options.module) |mod| {
- const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
- mod.main_pkg.root_src_path,
+ const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{
+ mod.main_mod.root_src_path,
});
_ = try man.addFile(main_zig_file, null);
{
- var seen_table = std.AutoHashMap(*Package, void).init(arena);
+ var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
// Skip builtin.zig; it is useless as an input, and we don't want to have to
// write it before checking for a cache hit.
- const builtin_pkg = mod.main_pkg.table.get("builtin").?;
- try seen_table.put(builtin_pkg, {});
+ const builtin_mod = mod.main_mod.deps.get("builtin").?;
+ try seen_table.put(builtin_mod, {});
- try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
+ try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man });
}
// Synchronize with other matching comments: ZigOnlyHashStuff
@@ -3564,8 +3562,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
decl.analysis = .codegen_failure_retryable;
};
},
- .analyze_pkg => |pkg| {
- const named_frame = tracy.namedFrame("analyze_pkg");
+ .analyze_mod => |pkg| {
+ const named_frame = tracy.namedFrame("analyze_mod");
defer named_frame.end();
const module = comp.bin_file.options.module.?;
@@ -6379,11 +6377,11 @@ fn buildOutputFromZig(
std.debug.assert(output_mode != .Exe);
- var main_pkg: Package = .{
+ var main_mod: Package = .{
.root_src_directory = comp.zig_lib_directory,
.root_src_path = src_basename,
};
- defer main_pkg.deinitTable(comp.gpa);
+ defer main_mod.deinitTable(comp.gpa);
const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
const target = comp.getTarget();
const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
@@ -6404,7 +6402,7 @@ fn buildOutputFromZig(
.cache_mode = .whole,
.target = target,
.root_name = root_name,
- .main_pkg = &main_pkg,
+ .main_mod = &main_mod,
.output_mode = output_mode,
.thread_pool = comp.thread_pool,
.libc_installation = comp.bin_file.options.libc_installation,
@@ -6481,7 +6479,7 @@ pub fn build_crt_file(
.cache_mode = .whole,
.target = target,
.root_name = root_name,
- .main_pkg = null,
+ .main_mod = null,
.output_mode = output_mode,
.thread_pool = comp.thread_pool,
.libc_installation = comp.bin_file.options.libc_installation,
src/crash_report.zig
@@ -139,18 +139,22 @@ fn dumpStatusReport() !void {
var crash_heap: [16 * 4096]u8 = undefined;
-fn writeFilePath(file: *Module.File, stream: anytype) !void {
- if (file.pkg.root_src_directory.path) |path| {
- try stream.writeAll(path);
- try stream.writeAll(std.fs.path.sep_str);
+fn writeFilePath(file: *Module.File, writer: anytype) !void {
+ if (file.mod.root.root_dir.path) |path| {
+ try writer.writeAll(path);
+ try writer.writeAll(std.fs.path.sep_str);
}
- try stream.writeAll(file.sub_file_path);
+ if (file.mod.root.sub_path.len > 0) {
+ try writer.writeAll(file.mod.root.sub_path);
+ try writer.writeAll(std.fs.path.sep_str);
+ }
+ try writer.writeAll(file.sub_file_path);
}
-fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, stream: anytype) !void {
- try writeFilePath(decl.getFileScope(mod), stream);
- try stream.writeAll(": ");
- try decl.renderFullyQualifiedDebugName(mod, stream);
+fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, writer: anytype) !void {
+ try writeFilePath(decl.getFileScope(mod), writer);
+ try writer.writeAll(": ");
+ try decl.renderFullyQualifiedDebugName(mod, writer);
}
pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn {
src/main.zig
@@ -416,7 +416,7 @@ const usage_build_generic =
\\ dep: [[import=]name]
\\ --deps [dep],[dep],... Set dependency names for the root package
\\ dep: [[import=]name]
- \\ --main-pkg-path Set the directory of the root package
+ \\ --main-mod-path Set the directory of the root module
\\ -fPIC Force-enable Position Independent Code
\\ -fno-PIC Force-disable Position Independent Code
\\ -fPIE Force-enable Position Independent Executable
@@ -765,17 +765,11 @@ const Framework = struct {
};
const CliModule = struct {
- mod: *Package,
+ mod: *Package.Module,
/// still in CLI arg format
deps_str: []const u8,
};
-fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void {
- var it = modules.iterator();
- while (it.next()) |kv| kv.value_ptr.mod.destroy(modules.allocator);
- modules.deinit();
-}
-
fn buildOutputType(
gpa: Allocator,
arena: Allocator,
@@ -950,8 +944,7 @@ fn buildOutputType(
// Contains every module specified via --mod. The dependencies are added
// after argument parsing is completed. We use a StringArrayHashMap to make
// error output consistent.
- var modules = std.StringArrayHashMap(CliModule).init(gpa);
- defer cleanupModules(&modules);
+ var modules = std.StringArrayHashMap(CliModule).init(arena);
// The dependency string for the root package
var root_deps_str: ?[]const u8 = null;
@@ -1023,32 +1016,37 @@ fn buildOutputType(
for ([_][]const u8{ "std", "root", "builtin" }) |name| {
if (mem.eql(u8, mod_name, name)) {
- fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src });
+ fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{
+ mod_name, root_src,
+ });
}
}
var mod_it = modules.iterator();
while (mod_it.next()) |kv| {
if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) {
- fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path });
+ fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{
+ mod_name, root_src, kv.value_ptr.mod.root_src_path,
+ });
}
}
- try modules.ensureUnusedCapacity(1);
- modules.put(mod_name, .{
- .mod = try Package.create(
- gpa,
- fs.path.dirname(root_src),
- fs.path.basename(root_src),
- ),
+ try modules.put(mod_name, .{
+ .mod = try Package.Module.create(arena, .{
+ .root = .{
+ .root_dir = Cache.Directory.cwd(),
+ .sub_path = fs.path.dirname(root_src) orelse "",
+ },
+ .root_src_path = fs.path.basename(root_src),
+ }),
.deps_str = deps_str,
- }) catch unreachable;
+ });
} else if (mem.eql(u8, arg, "--deps")) {
if (root_deps_str != null) {
fatal("only one --deps argument is allowed", .{});
}
root_deps_str = args_iter.nextOrFatal();
- } else if (mem.eql(u8, arg, "--main-pkg-path")) {
+ } else if (mem.eql(u8, arg, "--main-mod-path")) {
main_pkg_path = args_iter.nextOrFatal();
} else if (mem.eql(u8, arg, "-cflags")) {
extra_cflags.shrinkRetainingCapacity(0);
@@ -2461,19 +2459,26 @@ fn buildOutputType(
var deps_it = ModuleDepIterator.init(deps_str);
while (deps_it.next()) |dep| {
if (dep.expose.len == 0) {
- fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name });
+ fatal("module '{s}' depends on '{s}' with a blank name", .{
+ kv.key_ptr.*, dep.name,
+ });
}
for ([_][]const u8{ "std", "root", "builtin" }) |name| {
if (mem.eql(u8, dep.expose, name)) {
- fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose });
+ fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{
+ dep.name, dep.expose,
+ });
}
}
- const dep_mod = modules.get(dep.name) orelse
- fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name });
+ const dep_mod = modules.get(dep.name) orelse {
+ fatal("module '{s}' depends on module '{s}' which does not exist", .{
+ kv.key_ptr.*, dep.name,
+ });
+ };
- try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod);
+ try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod);
}
}
}
@@ -3229,31 +3234,33 @@ fn buildOutputType(
};
defer emit_implib_resolved.deinit();
- const main_pkg: ?*Package = if (root_src_file) |unresolved_src_path| blk: {
+ const main_mod: ?*Package.Module = if (root_src_file) |unresolved_src_path| blk: {
const src_path = try introspect.resolvePath(arena, unresolved_src_path);
if (main_pkg_path) |unresolved_main_pkg_path| {
const p = try introspect.resolvePath(arena, unresolved_main_pkg_path);
- if (p.len == 0) {
- break :blk try Package.create(gpa, null, src_path);
- } else {
- const rel_src_path = try fs.path.relative(arena, p, src_path);
- break :blk try Package.create(gpa, p, rel_src_path);
- }
+ break :blk try Package.Module.create(arena, .{
+ .root = .{
+ .root_dir = Cache.Directory.cwd(),
+ .sub_path = p,
+ },
+ .root_src_path = if (p.len == 0)
+ src_path
+ else
+ try fs.path.relative(arena, p, src_path),
+ });
} else {
- const root_src_dir_path = fs.path.dirname(src_path);
- break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch |err| {
- if (root_src_dir_path) |p| {
- fatal("unable to open '{s}': {s}", .{ p, @errorName(err) });
- } else {
- return err;
- }
- };
+ break :blk try Package.Module.create(arena, .{
+ .root = .{
+ .root_dir = Cache.Directory.cwd(),
+ .sub_path = fs.path.dirname(src_path) orelse "",
+ },
+ .root_src_path = fs.path.basename(src_path),
+ });
}
} else null;
- defer if (main_pkg) |p| p.destroy(gpa);
// Transfer packages added with --deps to the root package
- if (main_pkg) |mod| {
+ if (main_mod) |mod| {
var it = ModuleDepIterator.init(root_deps_str orelse "");
while (it.next()) |dep| {
if (dep.expose.len == 0) {
@@ -3269,7 +3276,7 @@ fn buildOutputType(
const dep_mod = modules.get(dep.name) orelse
fatal("root module depends on module '{s}' which does not exist", .{dep.name});
- try mod.add(gpa, dep.expose, dep_mod.mod);
+ try mod.deps.put(arena, dep.expose, dep_mod.mod);
}
}
@@ -3310,17 +3317,18 @@ fn buildOutputType(
if (arg_mode == .run) {
break :l global_cache_directory;
}
- if (main_pkg) |pkg| {
+ if (main_mod != null) {
// search upwards from cwd until we find directory with build.zig
const cwd_path = try process.getCwdAlloc(arena);
- const build_zig = "build.zig";
const zig_cache = "zig-cache";
var dirname: []const u8 = cwd_path;
while (true) {
- const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig });
+ const joined_path = try fs.path.join(arena, &.{
+ dirname, Package.build_zig_basename,
+ });
if (fs.cwd().access(joined_path, .{})) |_| {
- const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache });
- const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{});
+ const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache });
+ const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{});
cleanup_local_cache_dir = dir;
break :l .{ .handle = dir, .path = cache_dir_path };
} else |err| switch (err) {
@@ -3378,6 +3386,8 @@ fn buildOutputType(
gimmeMoreOfThoseSweetSweetFileDescriptors();
+ if (true) @panic("TODO restore Compilation logic");
+
const comp = Compilation.create(gpa, .{
.zig_lib_directory = zig_lib_directory,
.local_cache_directory = local_cache_directory,
@@ -3389,7 +3399,7 @@ fn buildOutputType(
.dynamic_linker = target_info.dynamic_linker.get(),
.sysroot = sysroot,
.output_mode = output_mode,
- .main_pkg = main_pkg,
+ .main_mod = main_mod,
.emit_bin = emit_bin_loc,
.emit_h = emit_h_resolved.data,
.emit_asm = emit_asm_resolved.data,
@@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
try thread_pool.init(.{ .allocator = gpa });
defer thread_pool.deinit();
- var cleanup_build_runner_dir: ?fs.Dir = null;
- defer if (cleanup_build_runner_dir) |*dir| dir.close();
-
- var main_pkg: Package = if (override_build_runner) |build_runner_path|
+ var main_mod: Package.Module = if (override_build_runner) |build_runner_path|
.{
- .root_src_directory = blk: {
- if (std.fs.path.dirname(build_runner_path)) |dirname| {
- const dir = fs.cwd().openDir(dirname, .{}) catch |err| {
- fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) });
- };
- cleanup_build_runner_dir = dir;
- break :blk .{ .path = dirname, .handle = dir };
- }
-
- break :blk .{ .path = null, .handle = fs.cwd() };
+ .root = .{
+ .root_dir = Cache.Directory.cwd(),
+ .sub_path = fs.path.dirname(build_runner_path) orelse "",
},
- .root_src_path = std.fs.path.basename(build_runner_path),
+ .root_src_path = fs.path.basename(build_runner_path),
}
else
.{
- .root_src_directory = zig_lib_directory,
+ .root = .{ .root_dir = zig_lib_directory },
.root_src_path = "build_runner.zig",
};
- var build_pkg: Package = .{
- .root_src_directory = build_directory,
+ var build_mod: Package.Module = .{
+ .root = .{ .root_dir = build_directory },
.root_src_path = build_zig_basename,
};
if (build_options.only_core_functionality) {
@@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
\\
);
- try main_pkg.add(gpa, "@dependencies", deps_pkg);
+ try main_mod.deps.put(arena, "@dependencies", deps_pkg);
} else {
var http_client: std.http.Client = .{ .allocator = gpa };
defer http_client.deinit();
+ if (true) @panic("TODO restore package fetching logic");
+
// Here we provide an import to the build runner that allows using reflection to find
// all of the dependencies. Without this, there would be no way to use `@import` to
// access dependencies by name, since `@import` requires string literals.
@@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
// Here we borrow main package's table and will replace it with a fresh
// one after this process completes.
- const fetch_result = build_pkg.fetchAndAddDependencies(
- &main_pkg,
+ const fetch_result = build_mod.fetchAndAddDependencies(
+ &main_mod,
arena,
&thread_pool,
&http_client,
@@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
dependencies_source.items,
);
- mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table);
- try main_pkg.add(gpa, "@dependencies", deps_pkg);
+ mem.swap(Package.Table, &main_mod.table, &deps_pkg.table);
+ try main_mod.add(gpa, "@dependencies", deps_pkg);
}
- try main_pkg.add(gpa, "@build", &build_pkg);
+ try main_mod.add(gpa, "@build", &build_mod);
const comp = Compilation.create(gpa, .{
.zig_lib_directory = zig_lib_directory,
@@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
.is_native_abi = cross_target.isNativeAbi(),
.dynamic_linker = target_info.dynamic_linker.get(),
.output_mode = .Exe,
- .main_pkg = &main_pkg,
+ .main_mod = &main_mod,
.emit_bin = emit_bin,
.emit_h = null,
.optimize_mode = .Debug,
@@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
.tree = tree,
.tree_loaded = true,
.zir = undefined,
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
- file.pkg = try Package.create(gpa, null, file.sub_file_path);
- defer file.pkg.destroy(gpa);
+ file.mod = try Package.Module.create(arena, .{
+ .root = Package.Path.cwd(),
+ .root_src_path = file.sub_file_path,
+ });
file.zir = try AstGen.generate(gpa, file.tree);
file.zir_loaded = true;
@@ -5321,12 +5325,14 @@ fn fmtPathFile(
.tree = tree,
.tree_loaded = true,
.zir = undefined,
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
- file.pkg = try Package.create(gpa, null, file.sub_file_path);
- defer file.pkg.destroy(gpa);
+ file.mod = try Package.Module.create(fmt.arena, .{
+ .root = Package.Path.cwd(),
+ .root_src_path = file.sub_file_path,
+ });
if (stat.size > max_src_size)
return error.FileTooBig;
@@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle(
tree: Ast,
path: []const u8,
wip_errors: *std.zig.ErrorBundle.Wip,
-) !void {
+) Allocator.Error!void {
var file: Module.File = .{
.status = .never_loaded,
.source_loaded = true,
@@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle(
.tree = tree,
.tree_loaded = true,
.zir = undefined,
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
- file.pkg = try Package.create(gpa, null, path);
- defer file.pkg.destroy(gpa);
+ file.mod = try Package.Module.create(gpa, .{
+ .root = Package.Path.cwd(),
+ .root_src_path = file.sub_file_path,
+ });
+ defer gpa.destroy(file.mod);
file.zir = try AstGen.generate(gpa, file.tree);
file.zir_loaded = true;
@@ -5933,7 +5942,7 @@ pub fn cmdAstCheck(
.stat = undefined,
.tree = undefined,
.zir = undefined,
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
if (zig_source_file) |file_name| {
@@ -5971,8 +5980,10 @@ pub fn cmdAstCheck(
file.stat.size = source.len;
}
- file.pkg = try Package.create(gpa, null, file.sub_file_path);
- defer file.pkg.destroy(gpa);
+ file.mod = try Package.Module.create(arena, .{
+ .root = Package.Path.cwd(),
+ .root_src_path = file.sub_file_path,
+ });
file.tree = try Ast.parse(gpa, file.source, .zig);
file.tree_loaded = true;
@@ -6067,7 +6078,7 @@ pub fn cmdDumpZir(
.stat = undefined,
.tree = undefined,
.zir = try Module.loadZirCache(gpa, f),
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
@@ -6136,12 +6147,14 @@ pub fn cmdChangelist(
},
.tree = undefined,
.zir = undefined,
- .pkg = undefined,
+ .mod = undefined,
.root_decl = .none,
};
- file.pkg = try Package.create(gpa, null, file.sub_file_path);
- defer file.pkg.destroy(gpa);
+ file.mod = try Package.Module.create(arena, .{
+ .root = Package.Path.cwd(),
+ .root_src_path = file.sub_file_path,
+ });
const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
const amt = try f.readAll(source);
@@ -6623,8 +6636,11 @@ fn cmdFetch(
args: []const []const u8,
) !void {
const color: Color = .auto;
- var opt_url: ?[]const u8 = null;
+ const work_around_btrfs_bug = builtin.os.tag == .linux and
+ std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
+ var opt_path_or_url: ?[]const u8 = null;
var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
+ var recursive = false;
{
var i: usize = 0;
@@ -6640,18 +6656,21 @@ fn cmdFetch(
i += 1;
override_global_cache_dir = args[i];
continue;
+ } else if (mem.eql(u8, arg, "--recursive")) {
+ recursive = true;
+ continue;
} else {
fatal("unrecognized parameter: '{s}'", .{arg});
}
- } else if (opt_url != null) {
+ } else if (opt_path_or_url != null) {
fatal("unexpected extra parameter: '{s}'", .{arg});
} else {
- opt_url = arg;
+ opt_path_or_url = arg;
}
}
}
- const url = opt_url orelse fatal("missing url or path parameter", .{});
+ const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
var thread_pool: ThreadPool = undefined;
try thread_pool.init(.{ .allocator = gpa });
@@ -6664,19 +6683,6 @@ fn cmdFetch(
const root_prog_node = progress.start("Fetch", 0);
defer root_prog_node.end();
- var wip_errors: std.zig.ErrorBundle.Wip = undefined;
- try wip_errors.init(gpa);
- defer wip_errors.deinit();
-
- var report: Package.Report = .{
- .ast = null,
- .directory = .{
- .handle = fs.cwd(),
- .path = null,
- },
- .error_bundle = &wip_errors,
- };
-
var global_cache_directory: Compilation.Directory = l: {
const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
break :l .{
@@ -6686,56 +6692,48 @@ fn cmdFetch(
};
defer global_cache_directory.handle.close();
- var readable_resource: Package.ReadableResource = rr: {
- if (fs.cwd().openIterableDir(url, .{})) |dir| {
- break :rr .{
- .path = try gpa.dupe(u8, url),
- .resource = .{ .dir = dir },
- };
- } else |dir_err| {
- const file_err = if (dir_err == error.NotDir) e: {
- if (fs.cwd().openFile(url, .{})) |f| {
- break :rr .{
- .path = try gpa.dupe(u8, url),
- .resource = .{ .file = f },
- };
- } else |err| break :e err;
- } else dir_err;
-
- const uri = std.Uri.parse(url) catch |uri_err| {
- fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
- url, @errorName(file_err), @errorName(uri_err),
- });
- };
- const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
- const cwd: Cache.Directory = .{
- .handle = fs.cwd(),
- .path = null,
- };
- break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
- }
+ var job_queue: Package.Fetch.JobQueue = .{
+ .http_client = &http_client,
+ .thread_pool = &thread_pool,
+ .global_cache = global_cache_directory,
+ .recursive = recursive,
+ .work_around_btrfs_bug = work_around_btrfs_bug,
+ };
+ defer job_queue.deinit();
+
+ var fetch: Package.Fetch = .{
+ .arena = std.heap.ArenaAllocator.init(gpa),
+ .location = .{ .path_or_url = path_or_url },
+ .location_tok = 0,
+ .hash_tok = 0,
+ .parent_package_root = undefined,
+ .parent_manifest_ast = null,
+ .prog_node = root_prog_node,
+ .job_queue = &job_queue,
+ .omit_missing_hash_error = true,
+
+ .package_root = undefined,
+ .error_bundle = undefined,
+ .manifest = null,
+ .manifest_ast = undefined,
+ .actual_hash = undefined,
+ .has_build_zig = false,
+ .oom_flag = false,
};
- defer readable_resource.deinit(gpa);
+ defer fetch.deinit();
- var package_location = readable_resource.unpack(
- gpa,
- &thread_pool,
- global_cache_directory,
- 0,
- report,
- root_prog_node,
- ) catch |err| {
- if (wip_errors.root_list.items.len > 0) {
- var errors = try wip_errors.toOwnedBundle("");
- defer errors.deinit(gpa);
- errors.renderToStdErr(renderOptions(color));
- process.exit(1);
- }
- fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
+ fetch.run() catch |err| switch (err) {
+ error.OutOfMemory => fatal("out of memory", .{}),
+ error.FetchFailed => {}, // error bundle checked below
};
- defer package_location.deinit(gpa);
- const hex_digest = Package.Manifest.hexDigest(package_location.hash);
+ if (fetch.error_bundle.root_list.items.len > 0) {
+ var errors = try fetch.error_bundle.toOwnedBundle("");
+ errors.renderToStdErr(renderOptions(color));
+ process.exit(1);
+ }
+
+ const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
progress.done = true;
progress.refresh();
src/Manifest.zig
@@ -1,6 +1,10 @@
pub const max_bytes = 10 * 1024 * 1024;
pub const basename = "build.zig.zon";
pub const Hash = std.crypto.hash.sha2.Sha256;
+pub const Digest = [Hash.digest_length]u8;
+pub const multihash_len = 1 + 1 + Hash.digest_length;
+pub const multihash_hex_digest_len = 2 * multihash_len;
+pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
pub const Dependency = struct {
location: union(enum) {
@@ -46,7 +50,6 @@ comptime {
assert(@intFromEnum(multihash_function) < 127);
assert(Hash.digest_length < 127);
}
-pub const multihash_len = 1 + 1 + Hash.digest_length;
name: []const u8,
version: std.SemanticVersion,
@@ -122,8 +125,8 @@ test hex64 {
try std.testing.expectEqualStrings("[00efcdab78563412]", s);
}
-pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
- var result: [multihash_len * 2]u8 = undefined;
+pub fn hexDigest(digest: Digest) MultiHashHexDigest {
+ var result: MultiHashHexDigest = undefined;
result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
@@ -339,10 +342,9 @@ const Parse = struct {
}
}
- const hex_multihash_len = 2 * Manifest.multihash_len;
- if (h.len != hex_multihash_len) {
+ if (h.len != multihash_hex_digest_len) {
return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
- hex_multihash_len, h.len,
+ multihash_hex_digest_len, h.len,
});
}
src/Module.zig
@@ -55,10 +55,10 @@ comp: *Compilation,
/// Where build artifacts and incremental compilation metadata serialization go.
zig_cache_artifact_directory: Compilation.Directory,
/// Pointer to externally managed resource.
-root_pkg: *Package,
-/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which
-/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests.
-main_pkg: *Package,
+root_mod: *Package.Module,
+/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which
+/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
+main_mod: *Package.Module,
sema_prog_node: std.Progress.Node = undefined,
/// Used by AstGen worker to load and store ZIR cache.
@@ -973,8 +973,8 @@ pub const File = struct {
tree: Ast,
/// Whether this is populated or not depends on `zir_loaded`.
zir: Zir,
- /// Package that this file is a part of, managed externally.
- pkg: *Package,
+ /// Module that this file is a part of, managed externally.
+ mod: *Package.Module,
/// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen.
multi_pkg: bool = false,
/// List of references to this file, used for multi-package errors.
@@ -1058,14 +1058,9 @@ pub const File = struct {
.stat = file.stat,
};
- const root_dir_path = file.pkg.root_src_directory.path orelse ".";
- log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
- root_dir_path, file.sub_file_path,
- });
-
// Keep track of inode, file size, mtime, hash so we can detect which files
// have been modified when an incremental update is requested.
- var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{});
+ var f = try file.mod.root.openFile(file.sub_file_path, .{});
defer f.close();
const stat = try f.stat();
@@ -1134,14 +1129,12 @@ pub const File = struct {
return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start);
}
- /// Returns the full path to this file relative to its package.
pub fn fullPath(file: File, ally: Allocator) ![]u8 {
- return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path});
+ return file.mod.root.joinString(ally, file.sub_file_path);
}
- /// Returns the full path to this file relative to its package.
pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 {
- return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path});
+ return file.mod.root.joinStringZ(ally, file.sub_file_path);
}
pub fn dumpSrc(file: *File, src: LazySrcLoc) void {
@@ -2543,25 +2536,25 @@ pub fn deinit(mod: *Module) void {
mod.deletion_set.deinit(gpa);
- // The callsite of `Compilation.create` owns the `main_pkg`, however
+ // The callsite of `Compilation.create` owns the `main_mod`, however
// Module owns the builtin and std packages that it adds.
- if (mod.main_pkg.table.fetchRemove("builtin")) |kv| {
+ if (mod.main_mod.table.fetchRemove("builtin")) |kv| {
gpa.free(kv.key);
kv.value.destroy(gpa);
}
- if (mod.main_pkg.table.fetchRemove("std")) |kv| {
+ if (mod.main_mod.table.fetchRemove("std")) |kv| {
gpa.free(kv.key);
- // It's possible for main_pkg to be std when running 'zig test'! In this case, we must not
+ // It's possible for main_mod to be std when running 'zig test'! In this case, we must not
// destroy it, since it would lead to a double-free.
- if (kv.value != mod.main_pkg) {
+ if (kv.value != mod.main_mod) {
kv.value.destroy(gpa);
}
}
- if (mod.main_pkg.table.fetchRemove("root")) |kv| {
+ if (mod.main_mod.table.fetchRemove("root")) |kv| {
gpa.free(kv.key);
}
- if (mod.root_pkg != mod.main_pkg) {
- mod.root_pkg.destroy(gpa);
+ if (mod.root_mod != mod.main_mod) {
+ mod.root_mod.destroy(gpa);
}
mod.compile_log_text.deinit(gpa);
@@ -2715,7 +2708,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
const stat = try source_file.stat();
- const want_local_cache = file.pkg == mod.main_pkg;
+ const want_local_cache = file.pkg == mod.main_mod;
const digest = hash: {
var path_hash: Cache.HashHelper = .{};
path_hash.addBytes(build_options.version);
@@ -3158,23 +3151,23 @@ pub fn populateBuiltinFile(mod: *Module) !void {
comp.mutex.lock();
defer comp.mutex.unlock();
- const builtin_pkg = mod.main_pkg.table.get("builtin").?;
- const result = try mod.importPkg(builtin_pkg);
+ const builtin_mod = mod.main_mod.table.get("builtin").?;
+ const result = try mod.importPkg(builtin_mod);
break :blk .{
.file = result.file,
- .pkg = builtin_pkg,
+ .pkg = builtin_mod,
};
};
const file = pkg_and_file.file;
- const builtin_pkg = pkg_and_file.pkg;
+ const builtin_mod = pkg_and_file.pkg;
const gpa = mod.gpa;
file.source = try comp.generateBuiltinZigSource(gpa);
file.source_loaded = true;
- if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) |stat| {
+ if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) |stat| {
if (stat.size != file.source.len) {
- const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{
- builtin_pkg.root_src_path,
+ const full_path = try builtin_mod.root_src_directory.join(gpa, &.{
+ builtin_mod.root_src_path,
});
defer gpa.free(full_path);
@@ -3184,7 +3177,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
.{ full_path, file.source.len, stat.size },
);
- try writeBuiltinFile(file, builtin_pkg);
+ try writeBuiltinFile(file, builtin_mod);
} else {
file.stat = .{
.size = stat.size,
@@ -3198,7 +3191,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
error.PipeBusy => unreachable, // it's not a pipe
error.WouldBlock => unreachable, // not asking for non-blocking I/O
- error.FileNotFound => try writeBuiltinFile(file, builtin_pkg),
+ error.FileNotFound => try writeBuiltinFile(file, builtin_mod),
else => |e| return e,
}
@@ -3212,8 +3205,8 @@ pub fn populateBuiltinFile(mod: *Module) !void {
file.status = .success_zir;
}
-fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void {
- var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{});
+fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void {
+ var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{});
defer af.deinit();
try af.file.writeAll(file.source);
try af.finish();
@@ -3748,7 +3741,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
// TODO: figure out how this works under incremental changes to builtin.zig!
const builtin_type_target_index: InternPool.Index = blk: {
- const std_mod = mod.main_pkg.table.get("std").?;
+ const std_mod = mod.main_mod.table.get("std").?;
if (decl.getFileScope(mod).pkg != std_mod) break :blk .none;
// We're in the std module.
const std_file = (try mod.importPkg(std_mod)).file;
@@ -4100,13 +4093,13 @@ pub fn importFile(
import_string: []const u8,
) !ImportFileResult {
if (std.mem.eql(u8, import_string, "std")) {
- return mod.importPkg(mod.main_pkg.table.get("std").?);
+ return mod.importPkg(mod.main_mod.table.get("std").?);
}
if (std.mem.eql(u8, import_string, "builtin")) {
- return mod.importPkg(mod.main_pkg.table.get("builtin").?);
+ return mod.importPkg(mod.main_mod.table.get("builtin").?);
}
if (std.mem.eql(u8, import_string, "root")) {
- return mod.importPkg(mod.root_pkg);
+ return mod.importPkg(mod.root_mod);
}
if (cur_file.pkg.table.get(import_string)) |pkg| {
return mod.importPkg(pkg);
@@ -4462,14 +4455,14 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
// test decl with no name. Skip the part where we check against
// the test name filter.
if (!comp.bin_file.options.is_test) break :blk false;
- if (decl_pkg != mod.main_pkg) break :blk false;
+ if (decl_pkg != mod.main_mod) break :blk false;
try mod.test_functions.put(gpa, new_decl_index, {});
break :blk true;
},
else => blk: {
if (!is_named_test) break :blk false;
if (!comp.bin_file.options.is_test) break :blk false;
- if (decl_pkg != mod.main_pkg) break :blk false;
+ if (decl_pkg != mod.main_mod) break :blk false;
if (comp.test_filter) |test_filter| {
if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) {
break :blk false;
@@ -5596,8 +5589,8 @@ pub fn populateTestFunctions(
) !void {
const gpa = mod.gpa;
const ip = &mod.intern_pool;
- const builtin_pkg = mod.main_pkg.table.get("builtin").?;
- const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;
+ const builtin_mod = mod.main_mod.table.get("builtin").?;
+ const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file;
const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?);
const builtin_namespace = mod.namespacePtr(root_decl.src_namespace);
const test_functions_str = try ip.getOrPutString(gpa, "test_functions");
src/Package.zig
@@ -1,251 +1,87 @@
-const Package = @This();
-
-const builtin = @import("builtin");
-const std = @import("std");
-const fs = std.fs;
-const mem = std.mem;
-const Allocator = mem.Allocator;
-const ascii = std.ascii;
-const assert = std.debug.assert;
-const log = std.log.scoped(.package);
-const main = @import("main.zig");
-const ThreadPool = std.Thread.Pool;
-
-const Compilation = @import("Compilation.zig");
-const Module = @import("Module.zig");
-const Cache = std.Build.Cache;
-const build_options = @import("build_options");
-const Fetch = @import("Package/Fetch.zig");
-
+pub const Module = @import("Package/Module.zig");
+pub const Fetch = @import("Package/Fetch.zig");
pub const build_zig_basename = "build.zig";
pub const Manifest = @import("Manifest.zig");
-pub const Table = std.StringHashMapUnmanaged(*Package);
-
-root_src_directory: Compilation.Directory,
-/// Relative to `root_src_directory`. May contain path separators.
-root_src_path: []const u8,
-/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root'
-/// are not specified in every dependency table, but instead only in the table of `main_pkg`.
-/// `Module.importFile` is responsible for detecting these names and using the correct package.
-table: Table = .{},
-/// Whether to free `root_src_directory` on `destroy`.
-root_src_directory_owned: bool = false,
-
-/// Allocate a Package. No references to the slices passed are kept.
-pub fn create(
- gpa: Allocator,
- /// Null indicates the current working directory
- root_src_dir_path: ?[]const u8,
- /// Relative to root_src_dir_path
- root_src_path: []const u8,
-) !*Package {
- const ptr = try gpa.create(Package);
- errdefer gpa.destroy(ptr);
-
- const owned_dir_path = if (root_src_dir_path) |p| try gpa.dupe(u8, p) else null;
- errdefer if (owned_dir_path) |p| gpa.free(p);
-
- const owned_src_path = try gpa.dupe(u8, root_src_path);
- errdefer gpa.free(owned_src_path);
- ptr.* = .{
- .root_src_directory = .{
- .path = owned_dir_path,
- .handle = if (owned_dir_path) |p| try fs.cwd().openDir(p, .{}) else fs.cwd(),
- },
- .root_src_path = owned_src_path,
- .root_src_directory_owned = true,
- };
+pub const Path = struct {
+ root_dir: Cache.Directory,
+ /// The path, relative to the root dir, that this `Path` represents.
+ /// Empty string means the root_dir is the path.
+ sub_path: []const u8 = "",
- return ptr;
-}
-
-pub fn createWithDir(
- gpa: Allocator,
- directory: Compilation.Directory,
- /// Relative to `directory`. If null, means `directory` is the root src dir
- /// and is owned externally.
- root_src_dir_path: ?[]const u8,
- /// Relative to root_src_dir_path
- root_src_path: []const u8,
-) !*Package {
- const ptr = try gpa.create(Package);
- errdefer gpa.destroy(ptr);
-
- const owned_src_path = try gpa.dupe(u8, root_src_path);
- errdefer gpa.free(owned_src_path);
-
- if (root_src_dir_path) |p| {
- const owned_dir_path = try directory.join(gpa, &[1][]const u8{p});
- errdefer gpa.free(owned_dir_path);
-
- ptr.* = .{
- .root_src_directory = .{
- .path = owned_dir_path,
- .handle = try directory.handle.openDir(p, .{}),
- },
- .root_src_directory_owned = true,
- .root_src_path = owned_src_path,
- };
- } else {
- ptr.* = .{
- .root_src_directory = directory,
- .root_src_directory_owned = false,
- .root_src_path = owned_src_path,
- };
+ pub fn cwd() Path {
+ return .{ .root_dir = Cache.Directory.cwd() };
}
- return ptr;
-}
-/// Free all memory associated with this package. It does not destroy any packages
-/// inside its table; the caller is responsible for calling destroy() on them.
-pub fn destroy(pkg: *Package, gpa: Allocator) void {
- gpa.free(pkg.root_src_path);
-
- if (pkg.root_src_directory_owned) {
- // If root_src_directory.path is null then the handle is the cwd()
- // which shouldn't be closed.
- if (pkg.root_src_directory.path) |p| {
- gpa.free(p);
- pkg.root_src_directory.handle.close();
- }
+ pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path {
+ const parts: []const []const u8 =
+ if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+ return .{
+ .root_dir = p.root_dir,
+ .sub_path = try fs.path.join(allocator, parts),
+ };
}
- pkg.deinitTable(gpa);
- gpa.destroy(pkg);
-}
-
-/// Only frees memory associated with the table.
-pub fn deinitTable(pkg: *Package, gpa: Allocator) void {
- pkg.table.deinit(gpa);
-}
-
-pub fn add(pkg: *Package, gpa: Allocator, name: []const u8, package: *Package) !void {
- try pkg.table.ensureUnusedCapacity(gpa, 1);
- const name_dupe = try gpa.dupe(u8, name);
- pkg.table.putAssumeCapacityNoClobber(name_dupe, package);
-}
-
-/// Compute a readable name for the package. The returned name should be freed from gpa. This
-/// function is very slow, as it traverses the whole package hierarchy to find a path to this
-/// package. It should only be used for error output.
-pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 {
- // we'll do a breadth-first search from the root module to try and find a short name for this
- // module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is
- // just the first-found shortest path - a module may be children of arbitrarily many other
- // modules. This path may vary between executions due to hashmap iteration order, but that
- // doesn't matter too much.
- var node_arena = std.heap.ArenaAllocator.init(gpa);
- defer node_arena.deinit();
- const Parented = struct {
- parent: ?*const @This(),
- mod: *const Package,
- };
- const Queue = std.DoublyLinkedList(Parented);
- var to_check: Queue = .{};
-
- {
- const new = try node_arena.allocator().create(Queue.Node);
- new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } };
- to_check.prepend(new);
+ pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
+ const parts: []const []const u8 =
+ if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+ return p.root_dir.join(allocator, parts);
}
- if (mod.main_pkg != mod.root_pkg) {
- const new = try node_arena.allocator().create(Queue.Node);
- // TODO: once #12201 is resolved, we may want a way of indicating a different name for this
- new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } };
- to_check.prepend(new);
+ pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
+ const parts: []const []const u8 =
+ if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+ return p.root_dir.joinZ(allocator, parts);
}
- // set of modules we've already checked to prevent loops
- var checked = std.AutoHashMap(*const Package, void).init(gpa);
- defer checked.deinit();
-
- const linked = while (to_check.pop()) |node| {
- const check = &node.data;
-
- if (checked.contains(check.mod)) continue;
- try checked.put(check.mod, {});
-
- if (check.mod == target) break check;
-
- var it = check.mod.table.iterator();
- while (it.next()) |kv| {
- var new = try node_arena.allocator().create(Queue.Node);
- new.* = .{ .data = .{
- .parent = check,
- .mod = kv.value_ptr.*,
- } };
- to_check.prepend(new);
- }
- } else {
- // this can happen for e.g. @cImport packages
- return gpa.dupe(u8, "<unnamed>");
- };
-
- // we found a path to the module! unfortunately, we can only traverse *up* it, so we have to put
- // all the names into a buffer so we can then print them in order.
- var names = std.ArrayList([]const u8).init(gpa);
- defer names.deinit();
-
- var cur: *const Parented = linked;
- while (cur.parent) |parent| : (cur = parent) {
- // find cur's name in parent
- var it = parent.mod.table.iterator();
- const name = while (it.next()) |kv| {
- if (kv.value_ptr.* == cur.mod) {
- break kv.key_ptr.*;
- }
- } else unreachable;
- try names.append(name);
+ pub fn openFile(
+ p: Path,
+ sub_path: []const u8,
+ flags: fs.File.OpenFlags,
+ ) fs.File.OpenError!fs.File {
+ var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+ const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+ break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+ p.sub_path, sub_path,
+ }) catch return error.NameTooLong;
+ };
+ return p.root_dir.handle.openFile(joined_path, flags);
}
- // finally, print the names into a buffer!
- var buf = std.ArrayList(u8).init(gpa);
- defer buf.deinit();
- try buf.writer().writeAll("root");
- var i: usize = names.items.len;
- while (i > 0) {
- i -= 1;
- try buf.writer().print(".{s}", .{names.items[i]});
+ pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
+ var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+ const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+ break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+ p.sub_path, sub_path,
+ }) catch return error.NameTooLong;
+ };
+ return p.root_dir.handle.makeOpenPath(joined_path, opts);
}
- return buf.toOwnedSlice();
-}
-
-pub fn createFilePkg(
- gpa: Allocator,
- cache_directory: Compilation.Directory,
- basename: []const u8,
- contents: []const u8,
-) !*Package {
- const rand_int = std.crypto.random.int(u64);
- const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int);
- {
- var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
- defer tmp_dir.close();
- try tmp_dir.writeFile(basename, contents);
+ pub fn format(
+ self: Path,
+ comptime fmt_string: []const u8,
+ options: std.fmt.FormatOptions,
+ writer: anytype,
+ ) !void {
+ _ = options;
+ if (fmt_string.len > 0)
+ std.fmt.invalidFmtError(fmt_string, self);
+ if (self.root_dir.path) |p| {
+ try writer.writeAll(p);
+ try writer.writeAll(fs.path.sep_str);
+ }
+ if (self.sub_path.len > 0) {
+ try writer.writeAll(self.sub_path);
+ try writer.writeAll(fs.path.sep_str);
+ }
}
-
- var hh: Cache.HashHelper = .{};
- hh.addBytes(build_options.version);
- hh.addBytes(contents);
- const hex_digest = hh.final();
-
- const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest;
- try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path);
-
- return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
-}
-
-const hex_multihash_len = 2 * Manifest.multihash_len;
-const MultiHashHexDigest = [hex_multihash_len]u8;
-
-const DependencyModule = union(enum) {
- zig_pkg: *Package,
- non_zig_pkg: *Package,
};
-/// This is to avoid creating multiple modules for the same build.zig file.
-/// If the value is `null`, the package is a known dependency, but has not yet
-/// been fetched.
-pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule);
+
+const Package = @This();
+const builtin = @import("builtin");
+const std = @import("std");
+const fs = std.fs;
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const Cache = std.Build.Cache;
build.zig
@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {
.name = "check-case",
.root_source_file = .{ .path = "test/src/Cases.zig" },
.optimize = optimize,
- .main_pkg_path = .{ .path = "." },
+ .main_mod_path = .{ .path = "." },
});
check_case_exe.stack_size = stack_size;
check_case_exe.single_threaded = single_threaded;