Commit f68cda738a
Changed files (9)
lib/std/heap/arena_allocator.zig
@@ -24,7 +24,7 @@ pub const ArenaAllocator = struct {
};
pub fn allocator(self: *ArenaAllocator) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
const BufNode = std.SinglyLinkedList([]u8).Node;
@@ -47,7 +47,7 @@ pub const ArenaAllocator = struct {
const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16);
const big_enough_len = prev_len + actual_min_size;
const len = big_enough_len + big_enough_len / 2;
- const buf = try self.child_allocator.vtable.alloc(self.child_allocator.ptr, len, @alignOf(BufNode), 1, @returnAddress());
+ const buf = try self.child_allocator.rawAlloc(len, @alignOf(BufNode), 1, @returnAddress());
const buf_node = @ptrCast(*BufNode, @alignCast(@alignOf(BufNode), buf.ptr));
buf_node.* = BufNode{
.data = buf,
@@ -111,4 +111,16 @@ pub const ArenaAllocator = struct {
return error.OutOfMemory;
}
}
+
+ fn free(self: *ArenaAllocator, buf: []u8, buf_align: u29, ret_addr: usize) void {
+ _ = buf_align;
+ _ = ret_addr;
+
+ const cur_node = self.state.buffer_list.first orelse return;
+ const cur_buf = cur_node.data[@sizeOf(BufNode)..];
+
+ if (@ptrToInt(cur_buf.ptr) + self.state.end_index == @ptrToInt(buf.ptr) + buf.len) {
+ self.state.end_index -= buf.len;
+ }
+ }
};
lib/std/heap/general_purpose_allocator.zig
@@ -281,7 +281,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
};
pub fn allocator(self: *Self) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
fn bucketStackTrace(
@@ -388,7 +388,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
var it = self.large_allocations.iterator();
while (it.next()) |large| {
if (large.value_ptr.freed) {
- _ = self.backing_allocator.vtable.resize(self.backing_allocator.ptr, large.value_ptr.bytes, large.value_ptr.ptr_align, 0, 0, @returnAddress()) catch unreachable;
+ self.backing_allocator.rawFree(large.value_ptr.bytes, large.value_ptr.ptr_align, @returnAddress());
}
}
}
@@ -529,9 +529,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
if (config.retain_metadata and entry.value_ptr.freed) {
if (config.safety) {
reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
- // Recoverable if this is a free.
- if (new_size == 0)
- return @as(usize, 0);
@panic("Unrecoverable double free");
} else {
unreachable;
@@ -555,7 +552,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
// Do memory limit accounting with requested sizes rather than what backing_allocator returns
// because if we want to return error.OutOfMemory, we have to leave allocation untouched, and
- // that is impossible to guarantee after calling backing_allocator.vtable.resize.
+ // that is impossible to guarantee after calling backing_allocator.rawResize.
const prev_req_bytes = self.total_requested_bytes;
if (config.enable_memory_limit) {
const new_req_bytes = prev_req_bytes + new_size - entry.value_ptr.requested_size;
@@ -568,29 +565,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
self.total_requested_bytes = prev_req_bytes;
};
- const result_len = if (config.never_unmap and new_size == 0)
- 0
- else
- try self.backing_allocator.vtable.resize(self.backing_allocator.ptr, old_mem, old_align, new_size, len_align, ret_addr);
+ const result_len = try self.backing_allocator.rawResize(old_mem, old_align, new_size, len_align, ret_addr);
if (config.enable_memory_limit) {
entry.value_ptr.requested_size = new_size;
}
- if (result_len == 0) {
- if (config.verbose_log) {
- log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
- }
-
- if (!config.retain_metadata) {
- assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr)));
- } else {
- entry.value_ptr.freed = true;
- entry.value_ptr.captureStackTrace(ret_addr, .free);
- }
- return 0;
- }
-
if (config.verbose_log) {
log.info("large resize {d} bytes at {*} to {d}", .{
old_mem.len, old_mem.ptr, new_size,
@@ -601,6 +581,64 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
return result_len;
}
+ /// This function assumes the object is in the large object storage regardless
+ /// of the parameters.
+ fn freeLarge(
+ self: *Self,
+ old_mem: []u8,
+ old_align: u29,
+ ret_addr: usize,
+ ) void {
+ _ = old_align;
+
+ const entry = self.large_allocations.getEntry(@ptrToInt(old_mem.ptr)) orelse {
+ if (config.safety) {
+ @panic("Invalid free");
+ } else {
+ unreachable;
+ }
+ };
+
+ if (config.retain_metadata and entry.value_ptr.freed) {
+ if (config.safety) {
+ reportDoubleFree(ret_addr, entry.value_ptr.getStackTrace(.alloc), entry.value_ptr.getStackTrace(.free));
+ return;
+ } else {
+ unreachable;
+ }
+ }
+
+ if (config.safety and old_mem.len != entry.value_ptr.bytes.len) {
+ var addresses: [stack_n]usize = [1]usize{0} ** stack_n;
+ var free_stack_trace = StackTrace{
+ .instruction_addresses = &addresses,
+ .index = 0,
+ };
+ std.debug.captureStackTrace(ret_addr, &free_stack_trace);
+ log.err("Allocation size {d} bytes does not match free size {d}. Allocation: {s} Free: {s}", .{
+ entry.value_ptr.bytes.len,
+ old_mem.len,
+ entry.value_ptr.getStackTrace(.alloc),
+ free_stack_trace,
+ });
+ }
+
+ if (config.enable_memory_limit) {
+ self.total_requested_bytes -= entry.value_ptr.requested_size;
+ }
+
+ if (config.verbose_log) {
+ log.info("large free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
+ }
+
+ if (!config.retain_metadata) {
+ assert(self.large_allocations.remove(@ptrToInt(old_mem.ptr)));
+ } else {
+ entry.value_ptr.freed = true;
+ entry.value_ptr.captureStackTrace(ret_addr, .free);
+ }
+ }
+
pub fn setRequestedMemoryLimit(self: *Self, limit: usize) void {
self.requested_memory_limit = limit;
}
@@ -656,9 +694,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
if (!is_used) {
if (config.safety) {
reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
- // Recoverable if this is a free.
- if (new_size == 0)
- return @as(usize, 0);
@panic("Unrecoverable double free");
} else {
unreachable;
@@ -678,52 +713,6 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
self.total_requested_bytes = prev_req_bytes;
};
- if (new_size == 0) {
- // Capture stack trace to be the "first free", in case a double free happens.
- bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
-
- used_byte.* &= ~(@as(u8, 1) << used_bit_index);
- bucket.used_count -= 1;
- if (bucket.used_count == 0) {
- if (bucket.next == bucket) {
- // it's the only bucket and therefore the current one
- self.buckets[bucket_index] = null;
- } else {
- bucket.next.prev = bucket.prev;
- bucket.prev.next = bucket.next;
- self.buckets[bucket_index] = bucket.prev;
- }
- if (!config.never_unmap) {
- self.backing_allocator.free(bucket.page[0..page_size]);
- }
- if (!config.retain_metadata) {
- self.freeBucket(bucket, size_class);
- } else {
- // move alloc_cursor to end so we can tell size_class later
- const slot_count = @divExact(page_size, size_class);
- bucket.alloc_cursor = @truncate(SlotIndex, slot_count);
- if (self.empty_buckets) |prev_bucket| {
- // empty_buckets is ordered newest to oldest through prev so that if
- // config.never_unmap is false and backing_allocator reuses freed memory
- // then searchBuckets will always return the newer, relevant bucket
- bucket.prev = prev_bucket;
- bucket.next = prev_bucket.next;
- prev_bucket.next = bucket;
- bucket.next.prev = bucket;
- } else {
- bucket.prev = bucket;
- bucket.next = bucket;
- }
- self.empty_buckets = bucket;
- }
- } else {
- @memset(old_mem.ptr, undefined, old_mem.len);
- }
- if (config.verbose_log) {
- log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
- }
- return @as(usize, 0);
- }
const new_aligned_size = math.max(new_size, old_align);
const new_size_class = math.ceilPowerOfTwoAssert(usize, new_aligned_size);
if (new_size_class <= size_class) {
@@ -740,6 +729,114 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
return error.OutOfMemory;
}
+ fn free(
+ self: *Self,
+ old_mem: []u8,
+ old_align: u29,
+ ret_addr: usize,
+ ) void {
+ const held = self.mutex.acquire();
+ defer held.release();
+
+ assert(old_mem.len != 0);
+
+ const aligned_size = math.max(old_mem.len, old_align);
+ if (aligned_size > largest_bucket_object_size) {
+ self.freeLarge(old_mem, old_align, ret_addr);
+ return;
+ }
+ const size_class_hint = math.ceilPowerOfTwoAssert(usize, aligned_size);
+
+ var bucket_index = math.log2(size_class_hint);
+ var size_class: usize = size_class_hint;
+ const bucket = while (bucket_index < small_bucket_count) : (bucket_index += 1) {
+ if (searchBucket(self.buckets[bucket_index], @ptrToInt(old_mem.ptr))) |bucket| {
+ // move bucket to head of list to optimize search for nearby allocations
+ self.buckets[bucket_index] = bucket;
+ break bucket;
+ }
+ size_class *= 2;
+ } else blk: {
+ if (config.retain_metadata) {
+ if (!self.large_allocations.contains(@ptrToInt(old_mem.ptr))) {
+ // object not in active buckets or a large allocation, so search empty buckets
+ if (searchBucket(self.empty_buckets, @ptrToInt(old_mem.ptr))) |bucket| {
+ // bucket is empty so is_used below will always be false and we exit there
+ break :blk bucket;
+ } else {
+ @panic("Invalid free");
+ }
+ }
+ }
+ self.freeLarge(old_mem, old_align, ret_addr);
+ return;
+ };
+ const byte_offset = @ptrToInt(old_mem.ptr) - @ptrToInt(bucket.page);
+ const slot_index = @intCast(SlotIndex, byte_offset / size_class);
+ const used_byte_index = slot_index / 8;
+ const used_bit_index = @intCast(u3, slot_index % 8);
+ const used_byte = bucket.usedBits(used_byte_index);
+ const is_used = @truncate(u1, used_byte.* >> used_bit_index) != 0;
+ if (!is_used) {
+ if (config.safety) {
+ reportDoubleFree(ret_addr, bucketStackTrace(bucket, size_class, slot_index, .alloc), bucketStackTrace(bucket, size_class, slot_index, .free));
+ // Recoverable if this is a free.
+ return;
+ } else {
+ unreachable;
+ }
+ }
+
+ // Definitely an in-use small alloc now.
+ if (config.enable_memory_limit) {
+ self.total_requested_bytes -= old_mem.len;
+ }
+
+ // Capture stack trace to be the "first free", in case a double free happens.
+ bucket.captureStackTrace(ret_addr, size_class, slot_index, .free);
+
+ used_byte.* &= ~(@as(u8, 1) << used_bit_index);
+ bucket.used_count -= 1;
+ if (bucket.used_count == 0) {
+ if (bucket.next == bucket) {
+ // it's the only bucket and therefore the current one
+ self.buckets[bucket_index] = null;
+ } else {
+ bucket.next.prev = bucket.prev;
+ bucket.prev.next = bucket.next;
+ self.buckets[bucket_index] = bucket.prev;
+ }
+ if (!config.never_unmap) {
+ self.backing_allocator.free(bucket.page[0..page_size]);
+ }
+ if (!config.retain_metadata) {
+ self.freeBucket(bucket, size_class);
+ } else {
+ // move alloc_cursor to end so we can tell size_class later
+ const slot_count = @divExact(page_size, size_class);
+ bucket.alloc_cursor = @truncate(SlotIndex, slot_count);
+ if (self.empty_buckets) |prev_bucket| {
+ // empty_buckets is ordered newest to oldest through prev so that if
+ // config.never_unmap is false and backing_allocator reuses freed memory
+ // then searchBuckets will always return the newer, relevant bucket
+ bucket.prev = prev_bucket;
+ bucket.next = prev_bucket.next;
+ prev_bucket.next = bucket;
+ bucket.next.prev = bucket;
+ } else {
+ bucket.prev = bucket;
+ bucket.next = bucket;
+ }
+ self.empty_buckets = bucket;
+ }
+ } else {
+ @memset(old_mem.ptr, undefined, old_mem.len);
+ }
+ if (config.verbose_log) {
+ log.info("small free {d} bytes at {*}", .{ old_mem.len, old_mem.ptr });
+ }
+ }
+
// Returns true if an allocation of `size` bytes is within the specified
// limits if enable_memory_limit is true
fn isAllocationAllowed(self: *Self, size: usize) bool {
@@ -764,7 +861,7 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
const new_aligned_size = math.max(len, ptr_align);
if (new_aligned_size > largest_bucket_object_size) {
try self.large_allocations.ensureUnusedCapacity(self.backing_allocator, 1);
- const slice = try self.backing_allocator.vtable.alloc(self.backing_allocator.ptr, len, ptr_align, len_align, ret_addr);
+ const slice = try self.backing_allocator.rawAlloc(len, ptr_align, len_align, ret_addr);
const gop = self.large_allocations.getOrPutAssumeCapacity(@ptrToInt(slice.ptr));
if (config.retain_metadata and !config.never_unmap) {
lib/std/heap/log_to_writer_allocator.zig
@@ -18,7 +18,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
}
pub fn allocator(self: *Self) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
fn alloc(
@@ -29,7 +29,7 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
ra: usize,
) error{OutOfMemory}![]u8 {
self.writer.print("alloc : {}", .{len}) catch {};
- const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra);
+ const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra);
if (result) |_| {
self.writer.print(" success!\n", .{}) catch {};
} else |_| {
@@ -46,14 +46,12 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
len_align: u29,
ra: usize,
) error{OutOfMemory}!usize {
- if (new_len == 0) {
- self.writer.print("free : {}\n", .{buf.len}) catch {};
- } else if (new_len <= buf.len) {
+ if (new_len <= buf.len) {
self.writer.print("shrink: {} to {}\n", .{ buf.len, new_len }) catch {};
} else {
self.writer.print("expand: {} to {}", .{ buf.len, new_len }) catch {};
}
- if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| {
+ if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| {
if (new_len > buf.len) {
self.writer.print(" success!\n", .{}) catch {};
}
@@ -64,6 +62,16 @@ pub fn LogToWriterAllocator(comptime Writer: type) type {
return e;
}
}
+
+ fn free(
+ self: *Self,
+ buf: []u8,
+ buf_align: u29,
+ ra: usize,
+ ) void {
+ self.writer.print("free : {}\n", .{buf.len}) catch {};
+ self.parent_allocator.rawFree(buf, buf_align, ra);
+ }
};
}
lib/std/heap/logging_allocator.zig
@@ -33,7 +33,7 @@ pub fn ScopedLoggingAllocator(
}
pub fn allocator(self: *Self) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
// This function is required as the `std.log.log` function is not public
@@ -53,7 +53,7 @@ pub fn ScopedLoggingAllocator(
len_align: u29,
ra: usize,
) error{OutOfMemory}![]u8 {
- const result = self.parent_allocator.vtable.alloc(self.parent_allocator.ptr, len, ptr_align, len_align, ra);
+ const result = self.parent_allocator.rawAlloc(len, ptr_align, len_align, ra);
if (result) |_| {
logHelper(
success_log_level,
@@ -78,10 +78,8 @@ pub fn ScopedLoggingAllocator(
len_align: u29,
ra: usize,
) error{OutOfMemory}!usize {
- if (self.parent_allocator.vtable.resize(self.parent_allocator.ptr, buf, buf_align, new_len, len_align, ra)) |resized_len| {
- if (new_len == 0) {
- logHelper(success_log_level, "free - success - len: {}", .{buf.len});
- } else if (new_len <= buf.len) {
+ if (self.parent_allocator.rawResize(buf, buf_align, new_len, len_align, ra)) |resized_len| {
+ if (new_len <= buf.len) {
logHelper(
success_log_level,
"shrink - success - {} to {}, len_align: {}, buf_align: {}",
@@ -106,6 +104,16 @@ pub fn ScopedLoggingAllocator(
return err;
}
}
+
+ fn free(
+ self: *Self,
+ buf: []u8,
+ buf_align: u29,
+ ra: usize,
+ ) void {
+ self.parent_allocator.rawFree(buf, buf_align, ra);
+ logHelper(success_log_level, "free - len: {}", .{buf.len});
+ }
};
}
lib/std/mem/Allocator.zig
@@ -5,6 +5,7 @@ const assert = std.debug.assert;
const math = std.math;
const mem = std.mem;
const Allocator = @This();
+const builtin = @import("builtin");
pub const Error = error{OutOfMemory};
@@ -28,9 +29,6 @@ pub const VTable = struct {
/// length returned by `alloc` or `resize`. `buf_align` must equal the same value
/// that was passed as the `ptr_align` parameter to the original `alloc` call.
///
- /// Passing a `new_len` of 0 frees and invalidates the buffer such that it can no
- /// longer be passed to `resize`.
- ///
/// error.OutOfMemory can only be returned if `new_len` is greater than `buf.len`.
/// If `buf` cannot be expanded to accomodate `new_len`, then the allocation MUST be
/// unmodified and error.OutOfMemory MUST be returned.
@@ -40,36 +38,54 @@ pub const VTable = struct {
/// provide a way to modify the alignment of a pointer. Rather it provides an API for
/// accepting more bytes of memory from the allocator than requested.
///
- /// `new_len` must be greater than or equal to `len_align` and must be aligned by `len_align`.
+ /// `new_len` must be greater than zero, greater than or equal to `len_align` and must be aligned by `len_align`.
///
/// `ret_addr` is optionally provided as the first return address of the allocation call stack.
/// If the value is `0` it means no return address has been provided.
resize: fn (ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
+
+ /// Free and invalidate a buffer. `buf.len` must equal the most recent length returned by `alloc` or `resize`.
+ /// `buf_align` must equal the same value that was passed as the `ptr_align` parameter to the original `alloc` call.
+ ///
+ /// `ret_addr` is optionally provided as the first return address of the allocation call stack.
+ /// If the value is `0` it means no return address has been provided.
+ free: fn (ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void,
};
pub fn init(
pointer: anytype,
comptime allocFn: fn (ptr: @TypeOf(pointer), len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8,
comptime resizeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize,
+ comptime freeFn: fn (ptr: @TypeOf(pointer), buf: []u8, buf_align: u29, ret_addr: usize) void,
) Allocator {
const Ptr = @TypeOf(pointer);
- assert(@typeInfo(Ptr) == .Pointer); // Must be a pointer
- assert(@typeInfo(Ptr).Pointer.size == .One); // Must be a single-item pointer
+ const ptr_info = @typeInfo(Ptr);
+
+ assert(ptr_info == .Pointer); // Must be a pointer
+ assert(ptr_info.Pointer.size == .One); // Must be a single-item pointer
+
+ const alignment = ptr_info.Pointer.alignment;
+
const gen = struct {
fn alloc(ptr: *c_void, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
- const alignment = @typeInfo(Ptr).Pointer.alignment;
const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
- return allocFn(self, len, ptr_align, len_align, ret_addr);
+ return @call(.{ .modifier = .always_inline }, allocFn, .{ self, len, ptr_align, len_align, ret_addr });
}
fn resize(ptr: *c_void, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize {
- const alignment = @typeInfo(Ptr).Pointer.alignment;
+ assert(new_len != 0);
+ const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
+ return @call(.{ .modifier = .always_inline }, resizeFn, .{ self, buf, buf_align, new_len, len_align, ret_addr });
+ }
+ fn free(ptr: *c_void, buf: []u8, buf_align: u29, ret_addr: usize) void {
const self = @ptrCast(Ptr, @alignCast(alignment, ptr));
- return resizeFn(self, buf, buf_align, new_len, len_align, ret_addr);
+ @call(.{ .modifier = .always_inline }, freeFn, .{ self, buf, buf_align, ret_addr });
}
};
+
const vtable = VTable{
.alloc = gen.alloc,
.resize = gen.resize,
+ .free = gen.free,
};
return .{
@@ -100,6 +116,56 @@ pub fn NoResize(comptime AllocatorType: type) type {
};
}
+/// Set freeFn to `NoOpFree(AllocatorType).noOpFree` if free is a no-op.
+pub fn NoOpFree(comptime AllocatorType: type) type {
+ return struct {
+ pub fn noOpFree(
+ self: *AllocatorType,
+ buf: []u8,
+ buf_align: u29,
+ ret_addr: usize,
+ ) void {
+ _ = self;
+ _ = buf;
+ _ = buf_align;
+ _ = ret_addr;
+ }
+ };
+}
+
+/// Set freeFn to `PanicFree(AllocatorType).noOpFree` if free is not a supported operation.
+pub fn PanicFree(comptime AllocatorType: type) type {
+ return struct {
+ pub fn noOpFree(
+ self: *AllocatorType,
+ buf: []u8,
+ buf_align: u29,
+ ret_addr: usize,
+ ) void {
+ _ = self;
+ _ = buf;
+ _ = buf_align;
+ _ = ret_addr;
+ @panic("free is not a supported operation for the allocator: " ++ @typeName(AllocatorType));
+ }
+ };
+}
+
+/// This function is not intended to be called except from within the implementation of an Allocator
+pub inline fn rawAlloc(self: Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Error![]u8 {
+ return self.vtable.alloc(self.ptr, len, ptr_align, len_align, ret_addr);
+}
+
+/// This function is not intended to be called except from within the implementation of an Allocator
+pub inline fn rawResize(self: Allocator, buf: []u8, buf_align: u29, new_len: usize, len_align: u29, ret_addr: usize) Error!usize {
+ return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, ret_addr);
+}
+
+/// This function is not intended to be called except from within the implementation of an Allocator
+pub inline fn rawFree(self: Allocator, buf: []u8, buf_align: u29, ret_addr: usize) void {
+ return self.vtable.free(self.ptr, buf, buf_align, ret_addr);
+}
+
/// Realloc is used to modify the size or alignment of an existing allocation,
/// as well as to provide the allocator with an opportunity to move an allocation
/// to a better location.
@@ -133,8 +199,7 @@ fn reallocBytes(
/// Guaranteed to be >= 1.
/// Guaranteed to be a power of 2.
old_alignment: u29,
- /// If `new_byte_count` is 0 then this is a free and it is guaranteed that
- /// `old_mem.len != 0`.
+ /// `new_byte_count` must be greater than zero
new_byte_count: usize,
/// Guaranteed to be >= 1.
/// Guaranteed to be a power of 2.
@@ -147,18 +212,20 @@ fn reallocBytes(
return_address: usize,
) Error![]u8 {
if (old_mem.len == 0) {
- const new_mem = try self.vtable.alloc(self.ptr, new_byte_count, new_alignment, len_align, return_address);
+ const new_mem = try self.rawAlloc(new_byte_count, new_alignment, len_align, return_address);
// TODO: https://github.com/ziglang/zig/issues/4298
@memset(new_mem.ptr, undefined, new_byte_count);
return new_mem;
}
+ assert(new_byte_count > 0); // `new_byte_count` must greater than zero, this is a resize not a free
+
if (mem.isAligned(@ptrToInt(old_mem.ptr), new_alignment)) {
if (new_byte_count <= old_mem.len) {
const shrunk_len = self.shrinkBytes(old_mem, old_alignment, new_byte_count, len_align, return_address);
return old_mem.ptr[0..shrunk_len];
}
- if (self.vtable.resize(self.ptr, old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| {
+ if (self.rawResize(old_mem, old_alignment, new_byte_count, len_align, return_address)) |resized_len| {
assert(resized_len >= new_byte_count);
// TODO: https://github.com/ziglang/zig/issues/4298
@memset(old_mem.ptr + new_byte_count, undefined, resized_len - new_byte_count);
@@ -184,11 +251,11 @@ fn moveBytes(
) Error![]u8 {
assert(old_mem.len > 0);
assert(new_len > 0);
- const new_mem = try self.vtable.alloc(self.ptr, new_len, new_alignment, len_align, return_address);
+ const new_mem = try self.rawAlloc(new_len, new_alignment, len_align, return_address);
@memcpy(new_mem.ptr, old_mem.ptr, math.min(new_len, old_mem.len));
// TODO https://github.com/ziglang/zig/issues/4298
@memset(old_mem.ptr, undefined, old_mem.len);
- _ = self.shrinkBytes(old_mem, old_align, 0, 0, return_address);
+ self.rawFree(old_mem, old_align, return_address);
return new_mem;
}
@@ -207,7 +274,7 @@ pub fn destroy(self: Allocator, ptr: anytype) void {
const T = info.child;
if (@sizeOf(T) == 0) return;
const non_const_ptr = @intToPtr([*]u8, @ptrToInt(ptr));
- _ = self.shrinkBytes(non_const_ptr[0..@sizeOf(T)], info.alignment, 0, 0, @returnAddress());
+ self.rawFree(non_const_ptr[0..@sizeOf(T)], info.alignment, @returnAddress());
}
/// Allocates an array of `n` items of type `T` and sets all the
@@ -326,7 +393,7 @@ pub fn allocAdvancedWithRetAddr(
.exact => 0,
.at_least => size_of_T,
};
- const byte_slice = try self.vtable.alloc(self.ptr, byte_count, a, len_align, return_address);
+ const byte_slice = try self.rawAlloc(byte_count, a, len_align, return_address);
switch (exact) {
.exact => assert(byte_slice.len == byte_count),
.at_least => assert(byte_slice.len >= byte_count),
@@ -351,7 +418,7 @@ pub fn resize(self: Allocator, old_mem: anytype, new_n: usize) Error!@TypeOf(old
}
const old_byte_slice = mem.sliceAsBytes(old_mem);
const new_byte_count = math.mul(usize, @sizeOf(T), new_n) catch return Error.OutOfMemory;
- const rc = try self.vtable.resize(self.ptr, old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());
+ const rc = try self.rawResize(old_byte_slice, Slice.alignment, new_byte_count, 0, @returnAddress());
assert(rc == new_byte_count);
const new_byte_slice = old_byte_slice.ptr[0..new_byte_count];
return mem.bytesAsSlice(T, new_byte_slice);
@@ -465,6 +532,11 @@ pub fn alignedShrinkWithRetAddr(
if (new_n == old_mem.len)
return old_mem;
+ if (new_n == 0) {
+ self.free(old_mem);
+ return @as([*]align(new_alignment) T, undefined)[0..0];
+ }
+
assert(new_n < old_mem.len);
assert(new_alignment <= Slice.alignment);
@@ -489,7 +561,7 @@ pub fn free(self: Allocator, memory: anytype) void {
const non_const_ptr = @intToPtr([*]u8, @ptrToInt(bytes.ptr));
// TODO: https://github.com/ziglang/zig/issues/4298
@memset(non_const_ptr, undefined, bytes_len);
- _ = self.shrinkBytes(non_const_ptr[0..bytes_len], Slice.alignment, 0, 0, @returnAddress());
+ self.rawFree(non_const_ptr[0..bytes_len], Slice.alignment, @returnAddress());
}
/// Copies `m` to newly allocated memory. Caller owns the memory.
@@ -520,5 +592,5 @@ pub fn shrinkBytes(
return_address: usize,
) usize {
assert(new_len <= buf.len);
- return self.vtable.resize(self.ptr, buf, buf_align, new_len, len_align, return_address) catch unreachable;
+ return self.rawResize(buf, buf_align, new_len, len_align, return_address) catch unreachable;
}
lib/std/testing/failing_allocator.zig
@@ -41,7 +41,7 @@ pub const FailingAllocator = struct {
}
pub fn allocator(self: *FailingAllocator) mem.Allocator {
- return mem.Allocator.init(self, alloc, resize);
+ return mem.Allocator.init(self, alloc, resize, free);
}
fn alloc(
@@ -54,7 +54,7 @@ pub const FailingAllocator = struct {
if (self.index == self.fail_index) {
return error.OutOfMemory;
}
- const result = try self.internal_allocator.vtable.alloc(self.internal_allocator.ptr, len, ptr_align, len_align, return_address);
+ const result = try self.internal_allocator.rawAlloc(len, ptr_align, len_align, return_address);
self.allocated_bytes += result.len;
self.allocations += 1;
self.index += 1;
@@ -69,18 +69,26 @@ pub const FailingAllocator = struct {
len_align: u29,
ra: usize,
) error{OutOfMemory}!usize {
- const r = self.internal_allocator.vtable.resize(self.internal_allocator.ptr, old_mem, old_align, new_len, len_align, ra) catch |e| {
+ const r = self.internal_allocator.rawResize(old_mem, old_align, new_len, len_align, ra) catch |e| {
std.debug.assert(new_len > old_mem.len);
return e;
};
- if (new_len == 0) {
- self.deallocations += 1;
- self.freed_bytes += old_mem.len;
- } else if (r < old_mem.len) {
+ if (r < old_mem.len) {
self.freed_bytes += old_mem.len - r;
} else {
self.allocated_bytes += r - old_mem.len;
}
return r;
}
+
+ fn free(
+ self: *FailingAllocator,
+ old_mem: []u8,
+ old_align: u29,
+ ra: usize,
+ ) void {
+ self.internal_allocator.rawFree(old_mem, old_align, ra);
+ self.deallocations += 1;
+ self.freed_bytes += old_mem.len;
+ }
};
lib/std/heap.zig
@@ -132,10 +132,6 @@ const CAllocator = struct {
) Allocator.Error!usize {
_ = buf_align;
_ = return_address;
- if (new_len == 0) {
- alignedFree(buf.ptr);
- return 0;
- }
if (new_len <= buf.len) {
return mem.alignAllocLen(buf.len, new_len, len_align);
}
@@ -147,6 +143,17 @@ const CAllocator = struct {
}
return error.OutOfMemory;
}
+
+ fn free(
+ _: *c_void,
+ buf: []u8,
+ buf_align: u29,
+ return_address: usize,
+ ) void {
+ _ = buf_align;
+ _ = return_address;
+ alignedFree(buf.ptr);
+ }
};
/// Supports the full Allocator interface, including alignment, and exploiting
@@ -159,6 +166,7 @@ pub const c_allocator = Allocator{
const c_allocator_vtable = Allocator.VTable{
.alloc = CAllocator.alloc,
.resize = CAllocator.resize,
+ .free = CAllocator.free,
};
/// Asserts allocations are within `@alignOf(std.c.max_align_t)` and directly calls
@@ -173,6 +181,7 @@ pub const raw_c_allocator = Allocator{
const raw_c_allocator_vtable = Allocator.VTable{
.alloc = rawCAlloc,
.resize = rawCResize,
+ .free = rawCFree,
};
fn rawCAlloc(
@@ -199,16 +208,23 @@ fn rawCResize(
) Allocator.Error!usize {
_ = old_align;
_ = ret_addr;
- if (new_len == 0) {
- c.free(buf.ptr);
- return 0;
- }
if (new_len <= buf.len) {
return mem.alignAllocLen(buf.len, new_len, len_align);
}
return error.OutOfMemory;
}
+fn rawCFree(
+ _: *c_void,
+ buf: []u8,
+ old_align: u29,
+ ret_addr: usize,
+) void {
+ _ = old_align;
+ _ = ret_addr;
+ c.free(buf.ptr);
+}
+
/// This allocator makes a syscall directly for every allocation and free.
/// Thread-safe and lock-free.
pub const page_allocator = if (builtin.target.isWasm())
@@ -238,6 +254,7 @@ const PageAllocator = struct {
const vtable = Allocator.VTable{
.alloc = alloc,
.resize = resize,
+ .free = free,
};
fn alloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) error{OutOfMemory}![]u8 {
@@ -351,16 +368,6 @@ const PageAllocator = struct {
if (builtin.os.tag == .windows) {
const w = os.windows;
- if (new_size == 0) {
- // From the docs:
- // "If the dwFreeType parameter is MEM_RELEASE, this parameter
- // must be 0 (zero). The function frees the entire region that
- // is reserved in the initial allocation call to VirtualAlloc."
- // So we can only use MEM_RELEASE when actually releasing the
- // whole allocation.
- w.VirtualFree(buf_unaligned.ptr, 0, w.MEM_RELEASE);
- return 0;
- }
if (new_size <= buf_unaligned.len) {
const base_addr = @ptrToInt(buf_unaligned.ptr);
const old_addr_end = base_addr + buf_unaligned.len;
@@ -391,8 +398,6 @@ const PageAllocator = struct {
const ptr = @alignCast(mem.page_size, buf_unaligned.ptr + new_size_aligned);
// TODO: if the next_mmap_addr_hint is within the unmapped range, update it
os.munmap(ptr[0 .. buf_aligned_len - new_size_aligned]);
- if (new_size_aligned == 0)
- return 0;
return alignPageAllocLen(new_size_aligned, new_size, len_align);
}
@@ -400,6 +405,19 @@ const PageAllocator = struct {
// TODO: if the next_mmap_addr_hint is within the remapped range, update it
return error.OutOfMemory;
}
+
+ fn free(_: *c_void, buf_unaligned: []u8, buf_align: u29, return_address: usize) void {
+ _ = buf_align;
+ _ = return_address;
+
+ if (builtin.os.tag == .windows) {
+ os.windows.VirtualFree(buf_unaligned.ptr, 0, os.windows.MEM_RELEASE);
+ } else {
+ const buf_aligned_len = mem.alignForward(buf_unaligned.len, mem.page_size);
+ const ptr = @alignCast(mem.page_size, buf_unaligned.ptr);
+ os.munmap(ptr[0..buf_aligned_len]);
+ }
+ }
};
const WasmPageAllocator = struct {
@@ -412,6 +430,7 @@ const WasmPageAllocator = struct {
const vtable = Allocator.VTable{
.alloc = alloc,
.resize = resize,
+ .free = free,
};
const PageStatus = enum(u1) {
@@ -571,7 +590,21 @@ const WasmPageAllocator = struct {
const base = nPages(@ptrToInt(buf.ptr));
freePages(base + new_n, base + current_n);
}
- return if (new_len == 0) 0 else alignPageAllocLen(new_n * mem.page_size, new_len, len_align);
+ return alignPageAllocLen(new_n * mem.page_size, new_len, len_align);
+ }
+
+ fn free(
+ _: *c_void,
+ buf: []u8,
+ buf_align: u29,
+ return_address: usize,
+ ) void {
+ _ = buf_align;
+ _ = return_address;
+ const aligned_len = mem.alignForward(buf.len, mem.page_size);
+ const current_n = nPages(aligned_len);
+ const base = nPages(@ptrToInt(buf.ptr));
+ freePages(base, base + current_n);
}
};
@@ -588,7 +621,7 @@ pub const HeapAllocator = switch (builtin.os.tag) {
}
pub fn allocator(self: *HeapAllocator) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
pub fn deinit(self: *HeapAllocator) void {
@@ -644,10 +677,6 @@ pub const HeapAllocator = switch (builtin.os.tag) {
) error{OutOfMemory}!usize {
_ = buf_align;
_ = return_address;
- if (new_size == 0) {
- os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*));
- return 0;
- }
const root_addr = getRecordPtr(buf).*;
const align_offset = @ptrToInt(buf.ptr) - root_addr;
@@ -669,6 +698,17 @@ pub const HeapAllocator = switch (builtin.os.tag) {
getRecordPtr(buf.ptr[0..return_len]).* = root_addr;
return return_len;
}
+
+ fn free(
+ self: *HeapAllocator,
+ buf: []u8,
+ buf_align: u29,
+ return_address: usize,
+ ) void {
+ _ = buf_align;
+ _ = return_address;
+ os.windows.HeapFree(self.heap_handle.?, 0, @intToPtr(*c_void, getRecordPtr(buf).*));
+ }
},
else => @compileError("Unsupported OS"),
};
@@ -696,13 +736,18 @@ pub const FixedBufferAllocator = struct {
/// *WARNING* using this at the same time as the interface returned by `threadSafeAllocator` is not thread safe
pub fn allocator(self: *FixedBufferAllocator) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
/// Provides a lock free thread safe `Allocator` interface to the underlying `FixedBufferAllocator`
/// *WARNING* using this at the same time as the interface returned by `getAllocator` is not thread safe
pub fn threadSafeAllocator(self: *FixedBufferAllocator) Allocator {
- return Allocator.init(self, threadSafeAlloc, Allocator.NoResize(FixedBufferAllocator).noResize);
+ return Allocator.init(
+ self,
+ threadSafeAlloc,
+ Allocator.NoResize(FixedBufferAllocator).noResize,
+ Allocator.NoOpFree(FixedBufferAllocator).noOpFree,
+ );
}
pub fn ownsPtr(self: *FixedBufferAllocator, ptr: [*]u8) bool {
@@ -715,7 +760,7 @@ pub const FixedBufferAllocator = struct {
/// NOTE: this will not work in all cases, if the last allocation had an adjusted_index
/// then we won't be able to determine what the last allocation was. This is because
- /// the alignForward operation done in alloc is not reverisible.
+ /// the alignForward operation done in alloc is not reversible.
pub fn isLastAllocation(self: *FixedBufferAllocator, buf: []u8) bool {
return buf.ptr + buf.len == self.buffer.ptr + self.end_index;
}
@@ -751,13 +796,13 @@ pub const FixedBufferAllocator = struct {
if (!self.isLastAllocation(buf)) {
if (new_size > buf.len)
return error.OutOfMemory;
- return if (new_size == 0) 0 else mem.alignAllocLen(buf.len, new_size, len_align);
+ return mem.alignAllocLen(buf.len, new_size, len_align);
}
if (new_size <= buf.len) {
const sub = buf.len - new_size;
self.end_index -= sub;
- return if (new_size == 0) 0 else mem.alignAllocLen(buf.len - sub, new_size, len_align);
+ return mem.alignAllocLen(buf.len - sub, new_size, len_align);
}
const add = new_size - buf.len;
@@ -768,6 +813,21 @@ pub const FixedBufferAllocator = struct {
return new_size;
}
+ fn free(
+ self: *FixedBufferAllocator,
+ buf: []u8,
+ buf_align: u29,
+ return_address: usize,
+ ) void {
+ _ = buf_align;
+ _ = return_address;
+ assert(self.ownsSlice(buf)); // sanity check
+
+ if (self.isLastAllocation(buf)) {
+ self.end_index -= buf.len;
+ }
+ }
+
fn threadSafeAlloc(self: *FixedBufferAllocator, n: usize, ptr_align: u29, len_align: u29, ra: usize) ![]u8 {
_ = len_align;
_ = ra;
@@ -810,7 +870,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
/// WARNING: This functions both fetches a `std.mem.Allocator` interface to this allocator *and* resets the internal buffer allocator
pub fn get(self: *Self) Allocator {
self.fixed_buffer_allocator = FixedBufferAllocator.init(self.buffer[0..]);
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
fn alloc(
@@ -821,7 +881,7 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
return_address: usize,
) error{OutOfMemory}![]u8 {
return FixedBufferAllocator.alloc(&self.fixed_buffer_allocator, len, ptr_align, len_align, return_address) catch
- return self.fallback_allocator.vtable.alloc(self.fallback_allocator.ptr, len, ptr_align, len_align, return_address);
+ return self.fallback_allocator.rawAlloc(len, ptr_align, len_align, return_address);
}
fn resize(
@@ -835,7 +895,20 @@ pub fn StackFallbackAllocator(comptime size: usize) type {
if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
return FixedBufferAllocator.resize(&self.fixed_buffer_allocator, buf, buf_align, new_len, len_align, return_address);
} else {
- return self.fallback_allocator.vtable.resize(self.fallback_allocator.ptr, buf, buf_align, new_len, len_align, return_address);
+ return self.fallback_allocator.rawResize(buf, buf_align, new_len, len_align, return_address);
+ }
+ }
+
+ fn free(
+ self: *Self,
+ buf: []u8,
+ buf_align: u29,
+ return_address: usize,
+ ) void {
+ if (self.fixed_buffer_allocator.ownsPtr(buf.ptr)) {
+ return FixedBufferAllocator.free(&self.fixed_buffer_allocator, buf, buf_align, return_address);
+ } else {
+ return self.fallback_allocator.rawFree(buf, buf_align, return_address);
}
}
};
lib/std/mem.zig
@@ -47,7 +47,7 @@ pub fn ValidationAllocator(comptime T: type) type {
}
pub fn allocator(self: *Self) Allocator {
- return Allocator.init(self, alloc, resize);
+ return Allocator.init(self, alloc, resize, free);
}
fn getUnderlyingAllocatorPtr(self: *Self) Allocator {
@@ -70,7 +70,7 @@ pub fn ValidationAllocator(comptime T: type) type {
}
const underlying = self.getUnderlyingAllocatorPtr();
- const result = try underlying.vtable.alloc(underlying.ptr, n, ptr_align, len_align, ret_addr);
+ const result = try underlying.rawAlloc(n, ptr_align, len_align, ret_addr);
assert(mem.isAligned(@ptrToInt(result.ptr), ptr_align));
if (len_align == 0) {
assert(result.len == n);
@@ -95,7 +95,7 @@ pub fn ValidationAllocator(comptime T: type) type {
assert(new_len >= len_align);
}
const underlying = self.getUnderlyingAllocatorPtr();
- const result = try underlying.vtable.resize(underlying.ptr, buf, buf_align, new_len, len_align, ret_addr);
+ const result = try underlying.rawResize(buf, buf_align, new_len, len_align, ret_addr);
if (len_align == 0) {
assert(result == new_len);
} else {
@@ -104,6 +104,19 @@ pub fn ValidationAllocator(comptime T: type) type {
}
return result;
}
+
+ pub fn free(
+ self: *Self,
+ buf: []u8,
+ buf_align: u29,
+ ret_addr: usize,
+ ) void {
+ _ = self;
+ _ = buf_align;
+ _ = ret_addr;
+ assert(buf.len > 0);
+ }
+
pub usingnamespace if (T == Allocator or !@hasDecl(T, "reset")) struct {} else struct {
pub fn reset(self: *Self) void {
self.underlying_allocator.reset();
@@ -139,6 +152,7 @@ const fail_allocator = Allocator{
const failAllocator_vtable = Allocator.VTable{
.alloc = failAllocatorAlloc,
.resize = Allocator.NoResize(c_void).noResize,
+ .free = Allocator.NoOpFree(c_void).noOpFree,
};
fn failAllocatorAlloc(_: *c_void, n: usize, alignment: u29, len_align: u29, ra: usize) Allocator.Error![]u8 {
src/tracy.zig
@@ -155,13 +155,10 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {
}
}
- if (resized_len != 0) {
- // this was a shrink or a resize
- if (name) |n| {
- allocNamed(buf.ptr, resized_len, n);
- } else {
- alloc(buf.ptr, resized_len);
- }
+ if (name) |n| {
+ allocNamed(buf.ptr, resized_len, n);
+ } else {
+ alloc(buf.ptr, resized_len);
}
return resized_len;
@@ -172,6 +169,15 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {
return err;
}
}
+
+ fn freeFn(self: *Self, buf: []u8, buf_align: u29, ret_addr: usize) void {
+ self.parent_allocator.rawFree(buf, buf_align, ret_addr);
+ if (name) |n| {
+ freeNamed(buf.ptr, n);
+ } else {
+ free(buf.ptr);
+ }
+ }
};
}