Commit 12355cfb4c
Changed files (4)
src
src/Package/Fetch.zig
@@ -56,7 +56,7 @@ package_root: Cache.Path,
error_bundle: ErrorBundle.Wip,
manifest: ?Manifest,
manifest_ast: std.zig.Ast,
-actual_hash: Manifest.Digest,
+computed_hash: ComputedHash,
/// Fetch logic notices whether a package has a build.zig file and sets this flag.
has_build_zig: bool,
/// Indicates whether the task aborted due to an out-of-memory condition.
@@ -116,8 +116,8 @@ pub const JobQueue = struct {
/// as lazy.
unlazy_set: UnlazySet = .{},
- pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
- pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);
+ pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
+ pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
pub fn deinit(jq: *JobQueue) void {
if (jq.all_fetches.items.len == 0) return;
@@ -160,22 +160,24 @@ pub const JobQueue = struct {
// Ensure the generated .zig file is deterministic.
jq.table.sortUnstable(@as(struct {
- keys: []const Manifest.MultiHashHexDigest,
+ keys: []const Package.Hash,
pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
- return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);
+ return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
}
}, .{ .keys = keys }));
- for (keys, jq.table.values()) |hash, fetch| {
+ for (keys, jq.table.values()) |*hash, fetch| {
if (fetch == jq.all_fetches.items[0]) {
// The first one is a dummy package for the current project.
continue;
}
+ const hash_slice = hash.toSlice();
+
try buf.writer().print(
\\ pub const {} = struct {{
\\
- , .{std.zig.fmtId(&hash)});
+ , .{std.zig.fmtId(hash_slice)});
lazy: {
switch (fetch.lazy_status) {
@@ -207,7 +209,7 @@ pub const JobQueue = struct {
try buf.writer().print(
\\ pub const build_zig = @import("{}");
\\
- , .{std.zig.fmtEscapes(&hash)});
+ , .{std.zig.fmtEscapes(hash_slice)});
}
if (fetch.manifest) |*manifest| {
@@ -219,7 +221,7 @@ pub const JobQueue = struct {
const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
try buf.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
- .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
+ .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
);
}
@@ -251,7 +253,7 @@ pub const JobQueue = struct {
const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
try buf.writer().print(
" .{{ \"{}\", \"{}\" }},\n",
- .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
+ .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
);
}
try buf.appendSlice("};\n");
@@ -283,7 +285,7 @@ pub const Location = union(enum) {
url: []const u8,
/// If this is null it means the user omitted the hash field from a dependency.
/// It will be an error but the logic should still fetch and print the discovered hash.
- hash: ?Manifest.MultiHashHexDigest,
+ hash: ?Package.Hash,
};
};
@@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void {
// "p/$hash/foo", with possibly more directories after "foo".
// We want to fail unless the resolved relative path has a
// prefix of "p/$hash/".
- const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
- const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];
+ const parent_sub_path = f.parent_package_root.sub_path;
+ const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
+ parent_sub_path.len;
+ const expected_prefix = parent_sub_path[prefix_len..end];
if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
return f.fail(
f.location_tok,
@@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void {
},
};
- const s = fs.path.sep_str;
if (remote.hash) |expected_hash| {
- const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
+ var prefixed_pkg_sub_path_buffer: [100]u8 = undefined;
+ prefixed_pkg_sub_path_buffer[0] = 'p';
+ prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
+ const hash_slice = expected_hash.toSlice();
+ @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
+ const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
@@ -437,7 +445,7 @@ fn runResource(
f: *Fetch,
uri_path: []const u8,
resource: *Resource,
- remote_hash: ?Manifest.MultiHashHexDigest,
+ remote_hash: ?Package.Hash,
) RunError!void {
defer resource.deinit();
const arena = f.arena.allocator();
@@ -499,7 +507,7 @@ fn runResource(
// Empty directories have already been omitted by `unpackResource`.
// Compute the package hash based on the remaining files in the temporary
// directory.
- f.actual_hash = try computeHash(f, pkg_path, filter);
+ f.computed_hash = try computeHash(f, pkg_path, filter);
break :blk if (unpack_result.root_dir.len > 0)
try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
@@ -507,6 +515,8 @@ fn runResource(
tmp_dir_sub_path;
};
+ const computed_package_hash = computedPackageHash(f);
+
// Rename the temporary directory into the global zig package cache
// directory. If the hash already exists, delete the temporary directory
// and leave the zig package cache directory untouched as it may be in use
@@ -515,7 +525,7 @@ fn runResource(
f.package_root = .{
.root_dir = cache_root,
- .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
+ .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
};
renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
@@ -534,13 +544,22 @@ fn runResource(
// Validate the computed hash against the expected hash. If invalid, this
// job is done.
- const actual_hex = Manifest.hexDigest(f.actual_hash);
if (remote_hash) |declared_hash| {
- if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
- return f.fail(f.hash_tok, try eb.printString(
- "hash mismatch: manifest declares {s} but the fetched package has {s}",
- .{ declared_hash, actual_hex },
- ));
+ if (declared_hash.isOld()) {
+ const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
+ if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
+ return f.fail(f.hash_tok, try eb.printString(
+ "hash mismatch: manifest declares {s} but the fetched package has {s}",
+ .{ declared_hash.toSlice(), actual_hex },
+ ));
+ }
+ } else {
+ if (!computed_package_hash.eql(&declared_hash)) {
+ return f.fail(f.hash_tok, try eb.printString(
+ "hash mismatch: manifest declares {s} but the fetched package has {s}",
+ .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
+ ));
+ }
}
} else if (!f.omit_missing_hash_error) {
const notes_len = 1;
@@ -551,7 +570,7 @@ fn runResource(
});
const notes_start = try eb.reserveNotes(notes_len);
eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
- .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
+ .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
}));
return error.FetchFailed;
}
@@ -562,6 +581,16 @@ fn runResource(
return queueJobsForDeps(f);
}
+pub fn computedPackageHash(f: *const Fetch) Package.Hash {
+ const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
+ if (f.manifest) |man| {
+ var version_buffer: [32]u8 = undefined;
+ const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
+ return .init(f.computed_hash.digest, man.name, version, saturated_size);
+ }
+ return .initNaked(f.computed_hash.digest, saturated_size);
+}
+
/// `computeHash` gets a free check for the existence of `build.zig`, but when
/// not computing a hash, we need to do a syscall to check for it.
fn checkBuildFileExistence(f: *Fetch) RunError!void {
@@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
.url = url,
.hash = h: {
const h = dep.hash orelse break :h null;
- const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
- const multihash_digest = h[0..digest_len].*;
- const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
+ const pkg_hash: Package.Hash = .fromSlice(h);
+ const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
if (gop.found_existing) {
if (!dep.lazy) {
gop.value_ptr.*.lazy_status = .eager;
@@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
continue;
}
gop.value_ptr.* = new_fetch;
- break :h multihash_digest;
+ break :h pkg_hash;
},
} },
.path => |rel_path| l: {
// This might produce an invalid path, which is checked for
// at the beginning of run().
const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
- const multihash_digest = relativePathDigest(new_root, cache_root);
- const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
+ const pkg_hash = relativePathDigest(new_root, cache_root);
+ const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
if (gop.found_existing) {
if (!dep.lazy) {
gop.value_ptr.*.lazy_status = .eager;
@@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
.error_bundle = undefined,
.manifest = null,
.manifest_ast = undefined,
- .actual_hash = undefined,
+ .computed_hash = undefined,
.has_build_zig = false,
.oom_flag = false,
.latest_commit = null,
@@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
}
}
-pub fn relativePathDigest(
- pkg_root: Cache.Path,
- cache_root: Cache.Directory,
-) Manifest.MultiHashHexDigest {
- var hasher = Manifest.Hash.init(.{});
+pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
+ var hasher = Package.Hash.Algo.init(.{});
// This hash is a tuple of:
// * whether it relative to the global cache directory or to the root package
// * the relative file path from there to the build root of the package
@@ -759,7 +784,7 @@ pub fn relativePathDigest(
else
&package_hash_prefix_project);
hasher.update(pkg_root.sub_path);
- return Manifest.hexDigest(hasher.finalResult());
+ return .fromSlice(&hasher.finalResult());
}
pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
@@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
}
}
-pub fn renameTmpIntoCache(
- cache_dir: fs.Dir,
- tmp_dir_sub_path: []const u8,
- dest_dir_sub_path: []const u8,
-) !void {
+pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
assert(dest_dir_sub_path[1] == fs.path.sep);
var handled_missing_dir = false;
while (true) {
@@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache(
}
}
+const ComputedHash = struct {
+ digest: Package.Hash.Digest,
+ total_size: u64,
+};
+
/// Assumes that files not included in the package have already been filtered
/// prior to calling this function. This ensures that files not protected by
/// the hash are not present on the file system. Empty directories are *not
/// hashed* and must not be present on the file system when calling this
/// function.
-fn computeHash(
- f: *Fetch,
- pkg_path: Cache.Path,
- filter: Filter,
-) RunError!Manifest.Digest {
+fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
// All the path name strings need to be in memory for sorting.
const arena = f.arena.allocator();
const gpa = f.arena.child_allocator;
@@ -1449,6 +1471,9 @@ fn computeHash(
var walker = try root_dir.walk(gpa);
defer walker.deinit();
+ // Total number of bytes of file contents included in the package.
+ var total_size: u64 = 0;
+
{
// The final hash will be a hash of each file hashed independently. This
// allows hashing in parallel.
@@ -1506,6 +1531,7 @@ fn computeHash(
.kind = kind,
.hash = undefined, // to be populated by the worker
.failure = undefined, // to be populated by the worker
+ .size = undefined, // to be populated by the worker
};
thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
try all_files.append(hashed_file);
@@ -1544,7 +1570,7 @@ fn computeHash(
std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
- var hasher = Manifest.Hash.init(.{});
+ var hasher = Package.Hash.Algo.init(.{});
var any_failures = false;
for (all_files.items) |hashed_file| {
hashed_file.failure catch |err| {
@@ -1556,6 +1582,7 @@ fn computeHash(
});
};
hasher.update(&hashed_file.hash);
+ total_size += hashed_file.size;
}
for (deleted_files.items) |deleted_file| {
deleted_file.failure catch |err| {
@@ -1580,7 +1607,10 @@ fn computeHash(
};
}
- return hasher.finalResult();
+ return .{
+ .digest = hasher.finalResult(),
+ .total_size = total_size,
+ };
}
fn dumpHashInfo(all_files: []const *const HashedFile) !void {
@@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
var buf: [8000]u8 = undefined;
- var hasher = Manifest.Hash.init(.{});
+ var hasher = Package.Hash.Algo.init(.{});
hasher.update(hashed_file.normalized_path);
+ var file_size: u64 = 0;
switch (hashed_file.kind) {
.file => {
@@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
while (true) {
const bytes_read = try file.read(&buf);
if (bytes_read == 0) break;
+ file_size += bytes_read;
hasher.update(buf[0..bytes_read]);
file_header.update(buf[0..bytes_read]);
}
@@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
},
}
hasher.final(&hashed_file.hash);
+ hashed_file.size = file_size;
}
fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
@@ -1667,9 +1700,10 @@ const DeletedFile = struct {
const HashedFile = struct {
fs_path: []const u8,
normalized_path: []const u8,
- hash: Manifest.Digest,
+ hash: Package.Hash.Digest,
failure: Error!void,
kind: Kind,
+ size: u64,
const Error =
fs.File.OpenError ||
@@ -1744,12 +1778,8 @@ const Filter = struct {
}
};
-pub fn depDigest(
- pkg_root: Cache.Path,
- cache_root: Cache.Directory,
- dep: Manifest.Dependency,
-) ?Manifest.MultiHashHexDigest {
- if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*;
+pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
+ if (dep.hash) |h| return .fromSlice(h);
switch (dep.location) {
.url => return null,
@@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" {
defer fb.deinit();
try fetch.run();
- const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+ const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
try std.testing.expectEqualStrings(
"12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
&hex_digest,
@@ -2181,7 +2211,7 @@ test "tarball without root folder" {
defer fb.deinit();
try fetch.run();
- const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+ const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
try std.testing.expectEqualStrings(
"12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
&hex_digest,
@@ -2222,7 +2252,7 @@ test "set executable bit based on file content" {
try fetch.run();
try std.testing.expectEqualStrings(
"1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
- &Manifest.hexDigest(fetch.actual_hash),
+ &Package.multiHashHexDigest(fetch.computed_hash.digest),
);
var out = try fb.packageDir();
@@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct {
.error_bundle = undefined,
.manifest = null,
.manifest_ast = undefined,
- .actual_hash = undefined,
+ .computed_hash = undefined,
.has_build_zig = false,
.oom_flag = false,
.latest_commit = null,
src/Package/Manifest.zig
@@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const Ast = std.zig.Ast;
const testing = std.testing;
-const hex_charset = std.fmt.hex_charset;
+const Package = @import("../Package.zig");
pub const max_bytes = 10 * 1024 * 1024;
pub const basename = "build.zig.zon";
-pub const Hash = std.crypto.hash.sha2.Sha256;
-pub const Digest = [Hash.digest_length]u8;
-pub const multihash_len = 1 + 1 + Hash.digest_length;
-pub const multihash_hex_digest_len = 2 * multihash_len;
-pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
pub const Dependency = struct {
location: Location,
@@ -38,35 +33,6 @@ pub const ErrorMessage = struct {
off: u32,
};
-pub const MultihashFunction = enum(u16) {
- identity = 0x00,
- sha1 = 0x11,
- @"sha2-256" = 0x12,
- @"sha2-512" = 0x13,
- @"sha3-512" = 0x14,
- @"sha3-384" = 0x15,
- @"sha3-256" = 0x16,
- @"sha3-224" = 0x17,
- @"sha2-384" = 0x20,
- @"sha2-256-trunc254-padded" = 0x1012,
- @"sha2-224" = 0x1013,
- @"sha2-512-224" = 0x1014,
- @"sha2-512-256" = 0x1015,
- @"blake2b-256" = 0xb220,
- _,
-};
-
-pub const multihash_function: MultihashFunction = switch (Hash) {
- std.crypto.hash.sha2.Sha256 => .@"sha2-256",
- else => @compileError("unreachable"),
-};
-comptime {
- // We avoid unnecessary uleb128 code in hexDigest by asserting here the
- // values are small enough to be contained in the one-byte encoding.
- assert(@intFromEnum(multihash_function) < 127);
- assert(Hash.digest_length < 127);
-}
-
name: []const u8,
version: std.SemanticVersion,
version_node: Ast.Node.Index,
@@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle(
}
}
-pub fn hexDigest(digest: Digest) MultiHashHexDigest {
- var result: MultiHashHexDigest = undefined;
-
- result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
- result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
-
- result[2] = hex_charset[Hash.digest_length >> 4];
- result[3] = hex_charset[Hash.digest_length & 15];
-
- for (digest, 0..) |byte, i| {
- result[4 + i * 2] = hex_charset[byte >> 4];
- result[5 + i * 2] = hex_charset[byte & 15];
- }
- return result;
-}
-
const Parse = struct {
gpa: Allocator,
ast: Ast,
@@ -421,21 +371,8 @@ const Parse = struct {
const tok = main_tokens[node];
const h = try parseString(p, node);
- if (h.len >= 2) {
- const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
- return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
- @errorName(err),
- });
- };
- if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
- return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
- }
- }
-
- if (h.len != multihash_hex_digest_len) {
- return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
- multihash_hex_digest_len, h.len,
- });
+ if (h.len > Package.Hash.max_len) {
+ return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
}
return h;
src/main.zig
@@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
.error_bundle = undefined,
.manifest = null,
.manifest_ast = undefined,
- .actual_hash = undefined,
+ .computed_hash = undefined,
.has_build_zig = true,
.oom_flag = false,
.latest_commit = null,
@@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
const hashes = job_queue.table.keys();
const fetches = job_queue.table.values();
try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
- for (hashes, fetches) |hash, f| {
+ for (hashes, fetches) |*hash, f| {
if (f == &fetch) {
// The first one is a dummy package for the current project.
continue;
}
if (!f.has_build_zig)
continue;
+ const hash_slice = hash.toSlice();
const m = try Package.Module.create(arena, .{
.global_cache_directory = global_cache_directory,
.paths = .{
@@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
.fully_qualified_name = try std.fmt.allocPrint(
arena,
"root.@dependencies.{s}",
- .{&hash},
+ .{hash_slice},
),
.cc_argv = &.{},
.inherited = .{},
@@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
.builtin_mod = builtin_mod,
.builtin_modules = null, // `builtin_mod` is specified
});
- const hash_cloned = try arena.dupe(u8, &hash);
+ const hash_cloned = try arena.dupe(u8, hash_slice);
deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
f.module = m;
}
@@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
var any_errors = false;
while (it.next()) |hash| {
if (hash.len == 0) continue;
- const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;
- if (hash.len != digest_len) {
- std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{
- hash.len, digest_len, hash,
+ if (hash.len > Package.Hash.max_len) {
+ std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
+ hash.len, hash,
});
any_errors = true;
continue;
}
- try unlazy_set.put(arena, hash[0..digest_len].*, {});
+ try unlazy_set.put(arena, .fromSlice(hash), {});
}
if (any_errors) process.exit(3);
if (system_pkg_dir_path) |p| {
// In this mode, the system needs to provide these packages; they
// cannot be fetched by Zig.
- for (unlazy_set.keys()) |hash| {
+ for (unlazy_set.keys()) |*hash| {
std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
- p, hash,
+ p, hash.toSlice(),
});
}
std.log.info("remote package fetching disabled due to --system mode", .{});
@@ -7097,7 +7097,7 @@ fn cmdFetch(
.error_bundle = undefined,
.manifest = null,
.manifest_ast = undefined,
- .actual_hash = undefined,
+ .computed_hash = undefined,
.has_build_zig = false,
.oom_flag = false,
.latest_commit = null,
@@ -7117,14 +7117,15 @@ fn cmdFetch(
process.exit(1);
}
- const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+ const package_hash = fetch.computedPackageHash();
+ const package_hash_slice = package_hash.toSlice();
root_prog_node.end();
root_prog_node = .{ .index = .none };
const name = switch (save) {
.no => {
- try io.getStdOut().writeAll(hex_digest ++ "\n");
+ try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
return cleanExit();
},
.yes, .exact => |name| name: {
@@ -7194,7 +7195,7 @@ fn cmdFetch(
\\ }}
, .{
std.zig.fmtEscapes(saved_path_or_url),
- std.zig.fmtEscapes(&hex_digest),
+ std.zig.fmtEscapes(package_hash_slice),
});
const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
@@ -7213,7 +7214,7 @@ fn cmdFetch(
if (dep.hash) |h| {
switch (dep.location) {
.url => |u| {
- if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {
+ if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
std.log.info("existing dependency named '{s}' is up-to-date", .{name});
process.exit(0);
}
@@ -7230,7 +7231,7 @@ fn cmdFetch(
const hash_replace = try std.fmt.allocPrint(
arena,
"\"{}\"",
- .{std.zig.fmtEscapes(&hex_digest)},
+ .{std.zig.fmtEscapes(package_hash_slice)},
);
warn("overwriting existing dependency named '{s}'", .{name});
src/Package.zig
@@ -1,8 +1,164 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
pub const Module = @import("Package/Module.zig");
pub const Fetch = @import("Package/Fetch.zig");
pub const build_zig_basename = "build.zig";
pub const Manifest = @import("Package/Manifest.zig");
+pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
+pub const multihash_hex_digest_len = 2 * multihash_len;
+pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
+
+/// A user-readable, file system safe hash that identifies an exact package
+/// snapshot, including file contents.
+///
+/// This data structure can be used to store the legacy hash format too. Legacy
+/// hash format is scheduled to be removed after 0.14.0 is tagged.
+pub const Hash = struct {
+ /// Maximum size of a package hash. Unused bytes at the end are
+ /// filled with zeroes.
+ bytes: [max_len]u8,
+
+ pub const Algo = std.crypto.hash.sha2.Sha256;
+ pub const Digest = [Algo.digest_length]u8;
+
+ pub const max_len = 32 + 1 + 32 + 1 + 12;
+
+ pub fn fromSlice(s: []const u8) Hash {
+ assert(s.len <= max_len);
+ var result: Hash = undefined;
+ @memcpy(result.bytes[0..s.len], s);
+ @memset(result.bytes[s.len..], 0);
+ return result;
+ }
+
+ pub fn toSlice(ph: *const Hash) []const u8 {
+ var end: usize = ph.bytes.len;
+ while (true) {
+ end -= 1;
+ if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
+ }
+ }
+
+ pub fn eql(a: *const Hash, b: *const Hash) bool {
+ return std.mem.eql(u8, &a.bytes, &b.bytes);
+ }
+
+ /// Distinguishes whether the legacy multihash format is being stored here.
+ pub fn isOld(h: *const Hash) bool {
+ if (h.bytes.len < 2) return false;
+ const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
+ if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
+ if (h.toSlice().len != multihash_hex_digest_len) return false;
+ return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
+ }
+
+ test isOld {
+ const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
+ try std.testing.expect(h.isOld());
+ }
+
+ /// Produces "$name-$semver-$sizedhash".
+ /// * name is the name field from build.zig.zon, truncated at 32 bytes and must
+ /// be a valid zig identifier
+ /// * semver is the version field from build.zig.zon, truncated at 32 bytes
+ /// * sizedhash is the following 9-byte array, base64 encoded using -_ to make
+ /// it filesystem safe:
+ /// - (4 bytes) LE u32 total decompressed size in bytes
+ /// - (5 bytes) truncated SHA-256 of hashed files of the package
+ ///
+ /// example: "nasm-2.16.1-2-BWdcABvF_jM1"
+ pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash {
+ var result: Hash = undefined;
+ var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
+ buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]);
+ buf.appendAssumeCapacity('-');
+ buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]);
+ buf.appendAssumeCapacity('-');
+ var sizedhash: [9]u8 = undefined;
+ std.mem.writeInt(u32, sizedhash[0..4], size, .little);
+ sizedhash[4..].* = digest[0..5].*;
+ _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash);
+ @memset(buf.unusedCapacitySlice(), 0);
+ return result;
+ }
+
+ /// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata.
+ /// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded
+ /// * the semver section is replaced with a hardcoded N which stands for
+ /// "naked". It acts as a version number so that any future updates to the
+ /// hash format can tell this hash format apart. Note that "N" is an
+ /// invalid semver.
+ /// * sizedhash is the same as in `init`.
+ ///
+ /// The hash is broken up this way so that "sizedhash" can be calculated
+ /// exactly the same way in both cases, and so that "name" and "hashiname" can
+ /// be used interchangeably in both cases.
+ pub fn initNaked(digest: Digest, size: u32) Hash {
+ var name: [32]u8 = undefined;
+ _ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]);
+ return init(digest, &name, "N", size);
+ }
+};
+
+pub const MultihashFunction = enum(u16) {
+ identity = 0x00,
+ sha1 = 0x11,
+ @"sha2-256" = 0x12,
+ @"sha2-512" = 0x13,
+ @"sha3-512" = 0x14,
+ @"sha3-384" = 0x15,
+ @"sha3-256" = 0x16,
+ @"sha3-224" = 0x17,
+ @"sha2-384" = 0x20,
+ @"sha2-256-trunc254-padded" = 0x1012,
+ @"sha2-224" = 0x1013,
+ @"sha2-512-224" = 0x1014,
+ @"sha2-512-256" = 0x1015,
+ @"blake2b-256" = 0xb220,
+ _,
+};
+
+pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
+ std.crypto.hash.sha2.Sha256 => .@"sha2-256",
+ else => @compileError("unreachable"),
+};
+
+pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
+ const hex_charset = std.fmt.hex_charset;
+
+ var result: MultiHashHexDigest = undefined;
+
+ result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
+ result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
+
+ result[2] = hex_charset[Hash.Algo.digest_length >> 4];
+ result[3] = hex_charset[Hash.Algo.digest_length & 15];
+
+ for (digest, 0..) |byte, i| {
+ result[4 + i * 2] = hex_charset[byte >> 4];
+ result[5 + i * 2] = hex_charset[byte & 15];
+ }
+ return result;
+}
+
+comptime {
+ // We avoid unnecessary uleb128 code in hexDigest by asserting here the
+ // values are small enough to be contained in the one-byte encoding.
+ assert(@intFromEnum(multihash_function) < 127);
+ assert(Hash.Algo.digest_length < 127);
+}
+
+test Hash {
+ const example_digest: Hash.Digest = .{
+ 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
+ 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
+ };
+ const result: Hash = .init(example_digest, "nasm", "2.16.1-2", 10 * 1024 * 1024);
+ try std.testing.expectEqualStrings("nasm-2.16.1-2-AACgAMf1cbe0", result.toSlice());
+}
+
test {
_ = Fetch;
}