Commit 390a4ded98
2021-06-29 23:29:59
1 parent
b0be0c7Changed files (7)
lib
std
doc/langref.html.in
@@ -10516,6 +10516,14 @@ fn readU32Be() u32 {}
See the Zig Standard Library for more examples.
</p>
{#header_close#}
+ {#header_open|Doc Comment Guidance#}
+ <ul>
+ <li>Omit any information that is redundant based on the name of the thing being documented.</li>
+ <li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
+ <li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
+ <li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
+ </ul>
+ {#header_close#}
{#header_close#}
{#header_open|Source Encoding#}
<p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>
lib/std/fs/file.zig
@@ -74,17 +74,28 @@ pub const File = struct {
read: bool = true,
write: bool = false,
- /// Open the file with a lock to prevent other processes from accessing it at the
- /// same time. An exclusive lock will prevent other processes from acquiring a lock.
- /// A shared lock will prevent other processes from acquiring a exclusive lock, but
- /// doesn't prevent other process from getting their own shared locks.
+ /// Open the file with an advisory lock to coordinate with other processes
+ /// accessing it at the same time. An exclusive lock will prevent other
+ /// processes from acquiring a lock. A shared lock will prevent other
+ /// processes from acquiring a exclusive lock, but does not prevent
+ /// other process from getting their own shared locks.
///
- /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
+ /// The lock is advisory, except on Linux in very specific cirsumstances[1].
/// This means that a process that does not respect the locking API can still get access
/// to the file, despite the lock.
///
- /// Windows' file locks are mandatory, and any process attempting to access the file will
- /// receive an error.
+ /// On these operating systems, the lock is acquired atomically with
+ /// opening the file:
+ /// * Darwin
+ /// * DragonFlyBSD
+ /// * FreeBSD
+ /// * Haiku
+ /// * NetBSD
+ /// * OpenBSD
+ /// On these operating systems, the lock is acquired via a separate syscall
+ /// after opening the file:
+ /// * Linux
+ /// * Windows
///
/// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
lock: Lock = .None,
@@ -120,17 +131,28 @@ pub const File = struct {
/// `error.PathAlreadyExists` to be returned.
exclusive: bool = false,
- /// Open the file with a lock to prevent other processes from accessing it at the
- /// same time. An exclusive lock will prevent other processes from acquiring a lock.
- /// A shared lock will prevent other processes from acquiring a exclusive lock, but
- /// doesn't prevent other process from getting their own shared locks.
+ /// Open the file with an advisory lock to coordinate with other processes
+ /// accessing it at the same time. An exclusive lock will prevent other
+ /// processes from acquiring a lock. A shared lock will prevent other
+ /// processes from acquiring a exclusive lock, but does not prevent
+ /// other process from getting their own shared locks.
///
- /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
+ /// The lock is advisory, except on Linux in very specific cirsumstances[1].
/// This means that a process that does not respect the locking API can still get access
/// to the file, despite the lock.
///
- /// Windows's file locks are mandatory, and any process attempting to access the file will
- /// receive an error.
+ /// On these operating systems, the lock is acquired atomically with
+ /// opening the file:
+ /// * Darwin
+ /// * DragonFlyBSD
+ /// * FreeBSD
+ /// * Haiku
+ /// * NetBSD
+ /// * OpenBSD
+ /// On these operating systems, the lock is acquired via a separate syscall
+ /// after opening the file:
+ /// * Linux
+ /// * Windows
///
/// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
lock: Lock = .None,
@@ -829,4 +851,165 @@ pub const File = struct {
pub fn seekableStream(file: File) SeekableStream {
return .{ .context = file };
}
+
+ const range_off: windows.LARGE_INTEGER = 0;
+ const range_len: windows.LARGE_INTEGER = 1;
+
+ pub const LockError = error{
+ SystemResources,
+ } || os.UnexpectedError;
+
+ /// Blocks when an incompatible lock is held by another process.
+ /// A process may hold only one type of lock (shared or exclusive) on
+ /// a file. When a process terminates in any way, the lock is released.
+ ///
+ /// Assumes the file is unlocked.
+ ///
+ /// TODO: integrate with async I/O
+ pub fn lock(file: File, l: Lock) LockError!void {
+ if (is_windows) {
+ var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+ const exclusive = switch (l) {
+ .None => return,
+ .Shared => false,
+ .Exclusive => true,
+ };
+ return windows.LockFile(
+ file.handle,
+ null,
+ null,
+ null,
+ &io_status_block,
+ &range_off,
+ &range_len,
+ null,
+ windows.FALSE, // non-blocking=false
+ @boolToInt(exclusive),
+ ) catch |err| switch (err) {
+ error.WouldBlock => unreachable, // non-blocking=false
+ else => |e| return e,
+ };
+ } else {
+ return os.flock(file.handle, switch (l) {
+ .None => os.LOCK_UN,
+ .Shared => os.LOCK_SH,
+ .Exclusive => os.LOCK_EX,
+ }) catch |err| switch (err) {
+ error.WouldBlock => unreachable, // non-blocking=false
+ else => |e| return e,
+ };
+ }
+ }
+
+ /// Assumes the file is locked.
+ pub fn unlock(file: File) void {
+ if (is_windows) {
+ var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+ return windows.UnlockFile(
+ file.handle,
+ &io_status_block,
+ &range_off,
+ &range_len,
+ null,
+ ) catch |err| switch (err) {
+ error.RangeNotLocked => unreachable, // Function assumes unlocked.
+ error.Unexpected => unreachable, // Resource deallocation must succeed.
+ };
+ } else {
+ return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
+ error.WouldBlock => unreachable, // unlocking can't block
+ error.SystemResources => unreachable, // We are deallocating resources.
+ error.Unexpected => unreachable, // Resource deallocation must succeed.
+ };
+ }
+ }
+
+ /// Attempts to obtain a lock, returning `true` if the lock is
+ /// obtained, and `false` if there was an existing incompatible lock held.
+ /// A process may hold only one type of lock (shared or exclusive) on
+ /// a file. When a process terminates in any way, the lock is released.
+ ///
+ /// Assumes the file is unlocked.
+ ///
+ /// TODO: integrate with async I/O
+ pub fn tryLock(file: File, l: Lock) LockError!bool {
+ if (is_windows) {
+ var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+ const exclusive = switch (l) {
+ .None => return,
+ .Shared => false,
+ .Exclusive => true,
+ };
+ windows.LockFile(
+ file.handle,
+ null,
+ null,
+ null,
+ &io_status_block,
+ &range_off,
+ &range_len,
+ null,
+ windows.TRUE, // non-blocking=true
+ @boolToInt(exclusive),
+ ) catch |err| switch (err) {
+ error.WouldBlock => return false,
+ else => |e| return e,
+ };
+ } else {
+ os.flock(file.handle, switch (l) {
+ .None => os.LOCK_UN,
+ .Shared => os.LOCK_SH | os.LOCK_NB,
+ .Exclusive => os.LOCK_EX | os.LOCK_NB,
+ }) catch |err| switch (err) {
+ error.WouldBlock => return false,
+ else => |e| return e,
+ };
+ }
+ return true;
+ }
+
+ /// Assumes the file is already locked in exclusive mode.
+ /// Atomically modifies the lock to be in shared mode, without releasing it.
+ ///
+ /// TODO: integrate with async I/O
+ pub fn downgradeLock(file: File) LockError!void {
+ if (is_windows) {
+ // On Windows it works like a semaphore + exclusivity flag. To implement this
+ // function, we first obtain another lock in shared mode. This changes the
+ // exclusivity flag, but increments the semaphore to 2. So we follow up with
+ // an NtUnlockFile which decrements the semaphore but does not modify the
+ // exclusivity flag.
+ var io_status_block: windows.IO_STATUS_BLOCK = undefined;
+ windows.LockFile(
+ file.handle,
+ null,
+ null,
+ null,
+ &io_status_block,
+ &range_off,
+ &range_len,
+ null,
+ windows.TRUE, // non-blocking=true
+ windows.FALSE, // exclusive=false
+ ) catch |err| switch (err) {
+ error.WouldBlock => unreachable, // File was not locked in exclusive mode.
+ else => |e| return e,
+ };
+ return windows.UnlockFile(
+ file.handle,
+ &io_status_block,
+ &range_off,
+ &range_len,
+ null,
+ ) catch |err| switch (err) {
+ error.RangeNotLocked => unreachable, // File was not locked.
+ error.Unexpected => unreachable, // Resource deallocation must succeed.
+ };
+ } else {
+ return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
+ error.WouldBlock => unreachable, // File was not locked in exclusive mode.
+ else => |e| return e,
+ };
+ }
+ }
};
lib/std/os/windows/ntdll.zig
@@ -121,3 +121,24 @@ pub extern "NtDll" fn NtQueryObject(
ObjectInformationLength: ULONG,
ReturnLength: ?*ULONG,
) callconv(WINAPI) NTSTATUS;
+
+pub extern "NtDll" fn NtLockFile(
+ FileHandle: HANDLE,
+ Event: ?HANDLE,
+ ApcRoutine: ?*IO_APC_ROUTINE,
+ ApcContext: ?*c_void,
+ IoStatusBlock: *IO_STATUS_BLOCK,
+ ByteOffset: *const LARGE_INTEGER,
+ Length: *const LARGE_INTEGER,
+ Key: ?*ULONG,
+ FailImmediately: BOOLEAN,
+ ExclusiveLock: BOOLEAN,
+) callconv(WINAPI) NTSTATUS;
+
+pub extern "NtDll" fn NtUnlockFile(
+ FileHandle: HANDLE,
+ IoStatusBlock: *IO_STATUS_BLOCK,
+ ByteOffset: *const LARGE_INTEGER,
+ Length: *const LARGE_INTEGER,
+ Key: ?*ULONG,
+) callconv(WINAPI) NTSTATUS;
lib/std/os/windows.zig
@@ -48,7 +48,6 @@ pub const OpenFileOptions = struct {
dir: ?HANDLE = null,
sa: ?*SECURITY_ATTRIBUTES = null,
share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
- share_access_nonblocking: bool = false,
creation: ULONG,
io_mode: std.io.ModeOverride,
/// If true, tries to open path as a directory.
@@ -59,8 +58,6 @@ pub const OpenFileOptions = struct {
follow_symlinks: bool = true,
};
-/// TODO when share_access_nonblocking is false, this implementation uses
-/// untinterruptible sleep() to block. This is not the final iteration of the API.
pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
return error.IsDir;
@@ -93,53 +90,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
// If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
- var delay: usize = 1;
- while (true) {
- const rc = ntdll.NtCreateFile(
- &result,
- options.access_mask,
- &attr,
- &io,
- null,
- FILE_ATTRIBUTE_NORMAL,
- options.share_access,
- options.creation,
- flags,
- null,
- 0,
- );
- switch (rc) {
- .SUCCESS => {
- if (std.io.is_async and options.io_mode == .evented) {
- _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
- }
- return result;
- },
- .OBJECT_NAME_INVALID => unreachable,
- .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
- .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
- .NO_MEDIA_IN_DEVICE => return error.NoDevice,
- .INVALID_PARAMETER => unreachable,
- .SHARING_VIOLATION => {
- if (options.share_access_nonblocking) {
- return error.WouldBlock;
- }
- // TODO sleep in a way that is interruptable
- // TODO integrate with async I/O
- std.time.sleep(delay);
- if (delay < 1 * std.time.ns_per_s) {
- delay *= 2;
- }
- continue;
- },
- .ACCESS_DENIED => return error.AccessDenied,
- .PIPE_BUSY => return error.PipeBusy,
- .OBJECT_PATH_SYNTAX_BAD => unreachable,
- .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
- .FILE_IS_A_DIRECTORY => return error.IsDir,
- .NOT_A_DIRECTORY => return error.NotDir,
- else => return unexpectedStatus(rc),
- }
+ const rc = ntdll.NtCreateFile(
+ &result,
+ options.access_mask,
+ &attr,
+ &io,
+ null,
+ FILE_ATTRIBUTE_NORMAL,
+ options.share_access,
+ options.creation,
+ flags,
+ null,
+ 0,
+ );
+ switch (rc) {
+ .SUCCESS => {
+ if (std.io.is_async and options.io_mode == .evented) {
+ _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
+ }
+ return result;
+ },
+ .OBJECT_NAME_INVALID => unreachable,
+ .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
+ .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
+ .NO_MEDIA_IN_DEVICE => return error.NoDevice,
+ .INVALID_PARAMETER => unreachable,
+ .SHARING_VIOLATION => return error.AccessDenied,
+ .ACCESS_DENIED => return error.AccessDenied,
+ .PIPE_BUSY => return error.PipeBusy,
+ .OBJECT_PATH_SYNTAX_BAD => unreachable,
+ .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
+ .FILE_IS_A_DIRECTORY => return error.IsDir,
+ .NOT_A_DIRECTORY => return error.NotDir,
+ else => return unexpectedStatus(rc),
}
}
@@ -1679,6 +1662,64 @@ pub fn SetFileTime(
}
}
+pub const LockFileError = error{
+ SystemResources,
+ WouldBlock,
+} || std.os.UnexpectedError;
+
+pub fn LockFile(
+ FileHandle: HANDLE,
+ Event: ?HANDLE,
+ ApcRoutine: ?*IO_APC_ROUTINE,
+ ApcContext: ?*c_void,
+ IoStatusBlock: *IO_STATUS_BLOCK,
+ ByteOffset: *const LARGE_INTEGER,
+ Length: *const LARGE_INTEGER,
+ Key: ?*ULONG,
+ FailImmediately: BOOLEAN,
+ ExclusiveLock: BOOLEAN,
+) !void {
+ const rc = ntdll.NtLockFile(
+ FileHandle,
+ Event,
+ ApcRoutine,
+ ApcContext,
+ IoStatusBlock,
+ ByteOffset,
+ Length,
+ Key,
+ FailImmediately,
+ ExclusiveLock,
+ );
+ switch (rc) {
+ .SUCCESS => return,
+ .INSUFFICIENT_RESOURCES => return error.SystemResources,
+ .LOCK_NOT_GRANTED => return error.WouldBlock,
+ .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
+ else => return unexpectedStatus(rc),
+ }
+}
+
+pub const UnlockFileError = error{
+ RangeNotLocked,
+} || std.os.UnexpectedError;
+
+pub fn UnlockFile(
+ FileHandle: HANDLE,
+ IoStatusBlock: *IO_STATUS_BLOCK,
+ ByteOffset: *const LARGE_INTEGER,
+ Length: *const LARGE_INTEGER,
+ Key: ?*ULONG,
+) !void {
+ const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
+ switch (rc) {
+ .SUCCESS => return,
+ .RANGE_NOT_LOCKED => return error.RangeNotLocked,
+ .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
+ else => return unexpectedStatus(rc),
+ }
+}
+
pub fn teb() *TEB {
return switch (builtin.target.cpu.arch) {
.i386 => asm volatile (
lib/std/fs.zig
@@ -883,24 +883,39 @@ pub const Dir = struct {
/// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
const w = os.windows;
- return @as(File, .{
- .handle = try os.windows.OpenFile(sub_path_w, .{
+ const file: File = .{
+ .handle = try w.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE |
(if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
(if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
- .share_access = switch (flags.lock) {
- .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
- .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
- .Exclusive => w.FILE_SHARE_DELETE,
- },
- .share_access_nonblocking = flags.lock_nonblocking,
.creation = w.FILE_OPEN,
.io_mode = flags.intended_io_mode,
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
- });
+ };
+ var io: w.IO_STATUS_BLOCK = undefined;
+ const range_off: w.LARGE_INTEGER = 0;
+ const range_len: w.LARGE_INTEGER = 1;
+ const exclusive = switch (flags.lock) {
+ .None => return file,
+ .Shared => false,
+ .Exclusive => true,
+ };
+ try w.LockFile(
+ file.handle,
+ null,
+ null,
+ null,
+ &io,
+ &range_off,
+ &range_len,
+ null,
+ @boolToInt(flags.lock_nonblocking),
+ @boolToInt(exclusive),
+ );
+ return file;
}
/// Creates, opens, or overwrites a file with write access.
@@ -1019,16 +1034,10 @@ pub const Dir = struct {
pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
const w = os.windows;
const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
- return @as(File, .{
+ const file: File = .{
.handle = try os.windows.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
- .share_access = switch (flags.lock) {
- .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
- .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
- .Exclusive => w.FILE_SHARE_DELETE,
- },
- .share_access_nonblocking = flags.lock_nonblocking,
.creation = if (flags.exclusive)
@as(u32, w.FILE_CREATE)
else if (flags.truncate)
@@ -1039,7 +1048,28 @@ pub const Dir = struct {
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
- });
+ };
+ var io: w.IO_STATUS_BLOCK = undefined;
+ const range_off: w.LARGE_INTEGER = 0;
+ const range_len: w.LARGE_INTEGER = 1;
+ const exclusive = switch (flags.lock) {
+ .None => return file,
+ .Shared => false,
+ .Exclusive => true,
+ };
+ try w.LockFile(
+ file.handle,
+ null,
+ null,
+ null,
+ &io,
+ &range_off,
+ &range_len,
+ null,
+ @boolToInt(flags.lock_nonblocking),
+ @boolToInt(exclusive),
+ );
+ return file;
}
pub const openRead = @compileError("deprecated in favor of openFile");
src/Cache.zig
@@ -181,6 +181,12 @@ pub const Manifest = struct {
hash: HashHelper,
manifest_file: ?fs.File,
manifest_dirty: bool,
+ /// Set this flag to true before calling hit() in order to indicate that
+ /// upon a cache hit, the code using the cache will not modify the files
+ /// within the cache directory. This allows multiple processes to utilize
+ /// the same cache directory at the same time.
+ want_shared_lock: bool = true,
+ have_exclusive_lock: bool = false,
files: std.ArrayListUnmanaged(File) = .{},
hex_digest: [hex_digest_len]u8,
/// Populated when hit() returns an error because of one
@@ -257,7 +263,9 @@ pub const Manifest = struct {
///
/// This function will also acquire an exclusive lock to the manifest file. This means
/// that a process holding a Manifest will block any other process attempting to
- /// acquire the lock.
+ /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
+ /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
+ /// file to be locked in exclusive mode.
///
/// The lock on the manifest file is released when `deinit` is called. As another
/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
@@ -285,31 +293,62 @@ pub const Manifest = struct {
mem.copy(u8, &manifest_file_path, &self.hex_digest);
manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
- if (self.files.items.len != 0) {
- self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
- .read = true,
- .truncate = false,
- .lock = .Exclusive,
- });
- } else {
+ if (self.files.items.len == 0) {
// If there are no file inputs, we check if the manifest file exists instead of
// comparing the hashes on the files used for the cached item
- self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ while (true) {
+ if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .read = true,
+ .write = true,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ break;
+ } else |open_err| switch (open_err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
+ });
+ break;
+ },
+ error.FileNotFound => {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
+ .read = true,
+ .truncate = false,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.manifest_dirty = true;
+ self.have_exclusive_lock = true;
+ return false; // cache miss; exclusive lock already held
+ } else |err| switch (err) {
+ error.WouldBlock => continue,
+ else => |e| return e,
+ }
+ },
+ else => |e| return e,
+ }
+ }
+ } else {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
.read = true,
- .write = true,
+ .truncate = false,
.lock = .Exclusive,
- }) catch |err| switch (err) {
- error.FileNotFound => {
- self.manifest_dirty = true;
- self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
- .read = true,
- .truncate = false,
- .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ } else |err| switch (err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
});
- return false;
},
else => |e| return e,
- };
+ }
}
const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
@@ -360,7 +399,10 @@ pub const Manifest = struct {
}
const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {
- error.FileNotFound => return false,
+ error.FileNotFound => {
+ try self.upgradeToExclusiveLock();
+ return false;
+ },
else => return error.CacheUnavailable,
};
defer this_file.close();
@@ -405,6 +447,7 @@ pub const Manifest = struct {
// cache miss
// keep the manifest file open
self.unhit(bin_digest, input_file_count);
+ try self.upgradeToExclusiveLock();
return false;
}
@@ -417,9 +460,11 @@ pub const Manifest = struct {
return err;
};
}
+ try self.upgradeToExclusiveLock();
return false;
}
+ try self.downgradeToSharedLock();
return true;
}
@@ -585,34 +630,58 @@ pub const Manifest = struct {
return out_digest;
}
+ /// If `want_shared_lock` is true, this function automatically downgrades the
+ /// lock from exclusive to shared.
pub fn writeManifest(self: *Manifest) !void {
const manifest_file = self.manifest_file.?;
- if (!self.manifest_dirty) return;
-
- var contents = std.ArrayList(u8).init(self.cache.gpa);
- defer contents.deinit();
+ if (self.manifest_dirty) {
+ self.manifest_dirty = false;
+
+ var contents = std.ArrayList(u8).init(self.cache.gpa);
+ defer contents.deinit();
+
+ const writer = contents.writer();
+ var encoded_digest: [hex_digest_len]u8 = undefined;
+
+ for (self.files.items) |file| {
+ _ = std.fmt.bufPrint(
+ &encoded_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
+ ) catch unreachable;
+ try writer.print("{d} {d} {d} {s} {s}\n", .{
+ file.stat.size,
+ file.stat.inode,
+ file.stat.mtime,
+ &encoded_digest,
+ file.path,
+ });
+ }
- const writer = contents.writer();
- var encoded_digest: [hex_digest_len]u8 = undefined;
+ try manifest_file.setEndPos(contents.items.len);
+ try manifest_file.pwriteAll(contents.items, 0);
+ }
- for (self.files.items) |file| {
- _ = std.fmt.bufPrint(
- &encoded_digest,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
- ) catch unreachable;
- try writer.print("{d} {d} {d} {s} {s}\n", .{
- file.stat.size,
- file.stat.inode,
- file.stat.mtime,
- &encoded_digest,
- file.path,
- });
+ if (self.want_shared_lock) {
+ try self.downgradeToSharedLock();
}
+ }
+
+ fn downgradeToSharedLock(self: *Manifest) !void {
+ if (!self.have_exclusive_lock) return;
+ const manifest_file = self.manifest_file.?;
+ try manifest_file.downgradeLock();
+ self.have_exclusive_lock = false;
+ }
- try manifest_file.setEndPos(contents.items.len);
- try manifest_file.pwriteAll(contents.items, 0);
- self.manifest_dirty = false;
+ fn upgradeToExclusiveLock(self: *Manifest) !void {
+ if (self.have_exclusive_lock) return;
+ const manifest_file = self.manifest_file.?;
+ // Here we intentionally have a period where the lock is released, in case there are
+ // other processes holding a shared lock.
+ manifest_file.unlock();
+ try manifest_file.lock(.Exclusive);
+ self.have_exclusive_lock = true;
}
/// Obtain only the data needed to maintain a lock on the manifest file.
@@ -881,27 +950,27 @@ test "no file inputs" {
defer cache.manifest_dir.close();
{
- var ch = cache.obtain();
- defer ch.deinit();
+ var man = cache.obtain();
+ defer man.deinit();
- ch.hash.addBytes("1234");
+ man.hash.addBytes("1234");
// There should be nothing in the cache
- try testing.expectEqual(false, try ch.hit());
+ try testing.expectEqual(false, try man.hit());
- digest1 = ch.final();
+ digest1 = man.final();
- try ch.writeManifest();
+ try man.writeManifest();
}
{
- var ch = cache.obtain();
- defer ch.deinit();
+ var man = cache.obtain();
+ defer man.deinit();
- ch.hash.addBytes("1234");
+ man.hash.addBytes("1234");
- try testing.expect(try ch.hit());
- digest2 = ch.final();
- try ch.writeManifest();
+ try testing.expect(try man.hit());
+ digest2 = man.final();
+ try man.writeManifest();
}
try testing.expectEqual(digest1, digest2);
src/Compilation.zig
@@ -39,7 +39,6 @@ gpa: *Allocator,
arena_state: std.heap.ArenaAllocator.State,
bin_file: *link.File,
c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
-c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
stage1_lock: ?Cache.Lock = null,
stage1_cache_manifest: *Cache.Manifest = undefined,
@@ -1570,7 +1569,6 @@ pub fn destroy(self: *Compilation) void {
key.destroy(gpa);
}
self.c_object_table.deinit(gpa);
- self.c_object_cache_digest_set.deinit(gpa);
for (self.failed_c_objects.values()) |value| {
value.destroy(gpa);
@@ -1607,7 +1605,6 @@ pub fn update(self: *Compilation) !void {
defer tracy.end();
self.clearMiscFailures();
- self.c_object_cache_digest_set.clearRetainingCapacity();
// For compiling C objects, we rely on the cache hash system to avoid duplicating work.
// Add a Job for each C object.
@@ -2566,25 +2563,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
try man.hashCSource(c_object.src);
- {
- const is_collision = blk: {
- const bin_digest = man.hash.peekBin();
-
- const lock = comp.mutex.acquire();
- defer lock.release();
-
- const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
- break :blk gop.found_existing;
- };
- if (is_collision) {
- return comp.failCObj(
- c_object,
- "the same source file was already added to the same compilation with the same flags",
- .{},
- );
- }
- }
-
var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
defer arena_allocator.deinit();
const arena = &arena_allocator.allocator;