Commit 3fff84a4a4
lib/std/zip/test.zig
@@ -1,298 +0,0 @@
-const std = @import("std");
-const testing = std.testing;
-const zip = @import("../zip.zig");
-const maxInt = std.math.maxInt;
-
-pub const File = struct {
- name: []const u8,
- content: []const u8,
- compression: zip.CompressionMethod,
-};
-
-pub fn expectFiles(
- test_files: []const File,
- dir: std.fs.Dir,
- opt: struct {
- strip_prefix: ?[]const u8 = null,
- },
-) !void {
- for (test_files) |test_file| {
- var normalized_sub_path_buf: [std.fs.max_path_bytes]u8 = undefined;
-
- const name = blk: {
- if (opt.strip_prefix) |strip_prefix| {
- try testing.expect(test_file.name.len >= strip_prefix.len);
- try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]);
- break :blk test_file.name[strip_prefix.len..];
- }
- break :blk test_file.name;
- };
- const normalized_sub_path = normalized_sub_path_buf[0..name.len];
- @memcpy(normalized_sub_path, name);
- std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
- var file = try dir.openFile(normalized_sub_path, .{});
- defer file.close();
- var content_buf: [4096]u8 = undefined;
- const n = try file.deprecatedReader().readAll(&content_buf);
- try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
- }
-}
-
-// Used to store any data from writing a file to the zip archive that's needed
-// when writing the corresponding central directory record.
-pub const FileStore = struct {
- compression: zip.CompressionMethod,
- file_offset: u64,
- crc32: u32,
- compressed_size: u32,
- uncompressed_size: usize,
-};
-
-pub fn makeZip(
- buf: []u8,
- comptime files: []const File,
- options: WriteZipOptions,
-) !std.io.FixedBufferStream([]u8) {
- var store: [files.len]FileStore = undefined;
- return try makeZipWithStore(buf, files, options, &store);
-}
-
-pub fn makeZipWithStore(
- buf: []u8,
- files: []const File,
- options: WriteZipOptions,
- store: []FileStore,
-) !std.io.FixedBufferStream([]u8) {
- var fbs = std.io.fixedBufferStream(buf);
- try writeZip(fbs.writer(), files, store, options);
- return std.io.fixedBufferStream(buf[0..fbs.pos]);
-}
-
-pub const WriteZipOptions = struct {
- end: ?EndRecordOptions = null,
- local_header: ?LocalHeaderOptions = null,
-};
-pub const LocalHeaderOptions = struct {
- zip64: ?LocalHeaderZip64Options = null,
- compressed_size: ?u32 = null,
- uncompressed_size: ?u32 = null,
- extra_len: ?u16 = null,
-};
-pub const LocalHeaderZip64Options = struct {
- data_size: ?u16 = null,
-};
-pub const EndRecordOptions = struct {
- zip64: ?Zip64Options = null,
- sig: ?[4]u8 = null,
- disk_number: ?u16 = null,
- central_directory_disk_number: ?u16 = null,
- record_count_disk: ?u16 = null,
- record_count_total: ?u16 = null,
- central_directory_size: ?u32 = null,
- central_directory_offset: ?u32 = null,
- comment_len: ?u16 = null,
- comment: ?[]const u8 = null,
-};
-pub const Zip64Options = struct {
- locator_sig: ?[4]u8 = null,
- locator_zip64_disk_count: ?u32 = null,
- locator_record_file_offset: ?u64 = null,
- locator_total_disk_count: ?u32 = null,
- //record_size: ?u64 = null,
- central_directory_size: ?u64 = null,
-};
-
-pub fn writeZip(
- writer: anytype,
- files: []const File,
- store: []FileStore,
- options: WriteZipOptions,
-) !void {
- if (store.len < files.len) return error.FileStoreTooSmall;
- var zipper = initZipper(writer);
- for (files, 0..) |file, i| {
- store[i] = try zipper.writeFile(.{
- .name = file.name,
- .content = file.content,
- .compression = file.compression,
- .write_options = options,
- });
- }
- for (files, 0..) |file, i| {
- try zipper.writeCentralRecord(store[i], .{
- .name = file.name,
- });
- }
- try zipper.writeEndRecord(if (options.end) |e| e else .{});
-}
-
-pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
- return .{ .counting_writer = std.io.countingWriter(writer) };
-}
-
-/// Provides methods to format and write the contents of a zip archive
-/// to the underlying Writer.
-pub fn Zipper(comptime Writer: type) type {
- return struct {
- counting_writer: std.io.CountingWriter(Writer),
- central_count: u64 = 0,
- first_central_offset: ?u64 = null,
- last_central_limit: ?u64 = null,
-
- const Self = @This();
-
- pub fn writeFile(
- self: *Self,
- opt: struct {
- name: []const u8,
- content: []const u8,
- compression: zip.CompressionMethod,
- write_options: WriteZipOptions,
- },
- ) !FileStore {
- const writer = self.counting_writer.writer();
-
- const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
- const crc32 = std.hash.Crc32.hash(opt.content);
-
- const header_options = opt.write_options.local_header;
- {
- var compressed_size: u32 = 0;
- var uncompressed_size: u32 = 0;
- var extra_len: u16 = 0;
- if (header_options) |hdr_options| {
- compressed_size = if (hdr_options.compressed_size) |size| size else 0;
- uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);
- extra_len = if (hdr_options.extra_len) |len| len else 0;
- }
- const hdr: zip.LocalFileHeader = .{
- .signature = zip.local_file_header_sig,
- .version_needed_to_extract = 10,
- .flags = .{ .encrypted = false, ._ = 0 },
- .compression_method = opt.compression,
- .last_modification_time = 0,
- .last_modification_date = 0,
- .crc32 = crc32,
- .compressed_size = compressed_size,
- .uncompressed_size = uncompressed_size,
- .filename_len = @intCast(opt.name.len),
- .extra_len = extra_len,
- };
- try writer.writeStructEndian(hdr, .little);
- }
- try writer.writeAll(opt.name);
-
- if (header_options) |hdr| {
- if (hdr.zip64) |options| {
- try writer.writeInt(u16, 0x0001, .little);
- const data_size = if (options.data_size) |size| size else 8;
- try writer.writeInt(u16, data_size, .little);
- try writer.writeInt(u64, 0, .little);
- try writer.writeInt(u64, @intCast(opt.content.len), .little);
- }
- }
-
- var compressed_size: u32 = undefined;
- switch (opt.compression) {
- .store => {
- try writer.writeAll(opt.content);
- compressed_size = @intCast(opt.content.len);
- },
- .deflate => {
- const offset = self.counting_writer.bytes_written;
- var fbs = std.io.fixedBufferStream(opt.content);
- try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
- std.debug.assert(fbs.pos == opt.content.len);
- compressed_size = @intCast(self.counting_writer.bytes_written - offset);
- },
- else => unreachable,
- }
- return .{
- .compression = opt.compression,
- .file_offset = file_offset,
- .crc32 = crc32,
- .compressed_size = compressed_size,
- .uncompressed_size = opt.content.len,
- };
- }
-
- pub fn writeCentralRecord(
- self: *Self,
- store: FileStore,
- opt: struct {
- name: []const u8,
- version_needed_to_extract: u16 = 10,
- },
- ) !void {
- if (self.first_central_offset == null) {
- self.first_central_offset = self.counting_writer.bytes_written;
- }
- self.central_count += 1;
-
- const hdr: zip.CentralDirectoryFileHeader = .{
- .signature = zip.central_file_header_sig,
- .version_made_by = 0,
- .version_needed_to_extract = opt.version_needed_to_extract,
- .flags = .{ .encrypted = false, ._ = 0 },
- .compression_method = store.compression,
- .last_modification_time = 0,
- .last_modification_date = 0,
- .crc32 = store.crc32,
- .compressed_size = store.compressed_size,
- .uncompressed_size = @intCast(store.uncompressed_size),
- .filename_len = @intCast(opt.name.len),
- .extra_len = 0,
- .comment_len = 0,
- .disk_number = 0,
- .internal_file_attributes = 0,
- .external_file_attributes = 0,
- .local_file_header_offset = @intCast(store.file_offset),
- };
- try self.counting_writer.writer().writeStructEndian(hdr, .little);
- try self.counting_writer.writer().writeAll(opt.name);
- self.last_central_limit = self.counting_writer.bytes_written;
- }
-
- pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
- const cd_offset = self.first_central_offset orelse 0;
- const cd_end = self.last_central_limit orelse 0;
-
- if (opt.zip64) |zip64| {
- const end64_off = cd_end;
- const fixed: zip.EndRecord64 = .{
- .signature = zip.end_record64_sig,
- .end_record_size = @sizeOf(zip.EndRecord64) - 12,
- .version_made_by = 0,
- .version_needed_to_extract = 45,
- .disk_number = 0,
- .central_directory_disk_number = 0,
- .record_count_disk = @intCast(self.central_count),
- .record_count_total = @intCast(self.central_count),
- .central_directory_size = @intCast(cd_end - cd_offset),
- .central_directory_offset = @intCast(cd_offset),
- };
- try self.counting_writer.writer().writeStructEndian(fixed, .little);
- const locator: zip.EndLocator64 = .{
- .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
- .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
- .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
- .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
- };
- try self.counting_writer.writer().writeStructEndian(locator, .little);
- }
- const hdr: zip.EndRecord = .{
- .signature = if (opt.sig) |s| s else zip.end_record_sig,
- .disk_number = if (opt.disk_number) |n| n else 0,
- .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
- .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
- .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
- .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
- .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
- .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
- };
- try self.counting_writer.writer().writeStructEndian(hdr, .little);
- if (opt.comment) |c|
- try self.counting_writer.writer().writeAll(c);
- }
- };
-}
lib/std/zip.zig
@@ -660,7 +660,3 @@ pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !voi
}
}
}
-
-test {
- _ = @import("zip/test.zig");
-}
src/Package/Fetch/git.zig
@@ -1564,9 +1564,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
defer pack_file.close();
try pack_file.writeAll(testrepo_pack);
+ var pack_file_buffer: [4096]u8 = undefined;
+ var pack_file_reader = pack_file.reader(&pack_file_buffer);
+
var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
defer index_file.close();
- try indexPack(testing.allocator, format, pack_file, index_file.deprecatedWriter());
+ try indexPack(testing.allocator, format, &pack_file_reader, index_file.deprecatedWriter());
// Arbitrary size limit on files read while checking the repository contents
// (all files in the test repo are known to be smaller than this)
@@ -1580,7 +1583,7 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
- var repository = try Repository.init(testing.allocator, format, pack_file, index_file);
+ var repository = try Repository.init(testing.allocator, format, &pack_file_reader, index_file);
defer repository.deinit();
var worktree = testing.tmpDir(.{ .iterate = true });
@@ -1673,6 +1676,9 @@ pub fn main() !void {
var pack_file = try std.fs.cwd().openFile(args[2], .{});
defer pack_file.close();
+ var pack_file_buffer: [4096]u8 = undefined;
+ var pack_file_reader = pack_file.reader(&pack_file_buffer);
+
const commit = try Oid.parse(format, args[3]);
var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});
defer worktree.close();
@@ -1684,11 +1690,11 @@ pub fn main() !void {
var index_file = try git_dir.createFile("idx", .{ .read = true });
defer index_file.close();
var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
- try indexPack(allocator, format, pack_file, index_buffered_writer.writer());
+ try indexPack(allocator, format, &pack_file_reader, index_buffered_writer.writer());
try index_buffered_writer.flush();
std.debug.print("Starting checkout...\n", .{});
- var repository = try Repository.init(allocator, format, pack_file, index_file);
+ var repository = try Repository.init(allocator, format, &pack_file_reader, index_file);
defer repository.deinit();
var diagnostics: Diagnostics = .{ .allocator = allocator };
defer diagnostics.deinit();
src/Package/Fetch.zig
@@ -2076,72 +2076,6 @@ const UnpackResult = struct {
}
};
-test "zip" {
- const gpa = std.testing.allocator;
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
-
- const test_files = [_]std.zip.testutil.File{
- .{ .name = "foo", .content = "this is just foo\n", .compression = .store },
- .{ .name = "bar", .content = "another file\n", .compression = .deflate },
- };
- {
- var zip_file = try tmp.dir.createFile("test.zip", .{});
- defer zip_file.close();
- var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
- var store: [test_files.len]std.zip.testutil.FileStore = undefined;
- try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
- try bw.flush();
- }
-
- const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
- defer gpa.free(zip_path);
-
- var fb: TestFetchBuilder = undefined;
- var fetch = try fb.build(gpa, tmp.dir, zip_path);
- defer fb.deinit();
-
- try fetch.run();
-
- var out = try fb.packageDir();
- defer out.close();
-
- try std.zip.testutil.expectFiles(&test_files, out, .{});
-}
-
-test "zip with one root folder" {
- const gpa = std.testing.allocator;
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
-
- const test_files = [_]std.zip.testutil.File{
- .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store },
- .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store },
- };
- {
- var zip_file = try tmp.dir.createFile("test.zip", .{});
- defer zip_file.close();
- var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
- var store: [test_files.len]std.zip.testutil.FileStore = undefined;
- try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
- try bw.flush();
- }
-
- const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
- defer gpa.free(zip_path);
-
- var fb: TestFetchBuilder = undefined;
- var fetch = try fb.build(gpa, tmp.dir, zip_path);
- defer fb.deinit();
-
- try fetch.run();
-
- var out = try fb.packageDir();
- defer out.close();
-
- try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" });
-}
-
test "tarball with duplicate paths" {
// This tarball has duplicate path 'dir1/file1' to simulate case sensitve
// file system on any file sytstem.