Commit 61c54b2d49
Changed files (6)
src-self-hosted
src-self-hosted/link/Elf.zig
@@ -1590,7 +1590,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
const ok = llvm.Link(.ELF, new_argv.ptr, new_argv.len, append_diagnostic, 0, 0);
if (!ok) return error.LLDReportedFailure;
- // Update the dangling symlink "id.txt" with the digest. If it fails we can continue; it only
+ // Update the dangling symlink with the digest. If it fails we can continue; it only
// means that the next invocation will have an unnecessary cache miss.
directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
src-self-hosted/Cache.zig
@@ -410,7 +410,7 @@ pub const CacheHash = struct {
/// calculated. This is useful for processes that don't know the all the files that
/// are depended on ahead of time. For example, a source file that can import other files
/// will need to be recompiled if the imported file is changed.
- pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
+ pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]const u8 {
assert(self.manifest_file != null);
const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
src-self-hosted/Compilation.zig
@@ -27,7 +27,8 @@ gpa: *Allocator,
arena_state: std.heap.ArenaAllocator.State,
bin_file: *link.File,
c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
-stage1_module: ?*stage1.Module,
+stage1_lock: ?Cache.Lock = null,
+stage1_cache_hash: *Cache.CacheHash = undefined,
link_error_flags: link.File.ErrorFlags = .{},
@@ -54,6 +55,7 @@ verbose_cimport: bool,
verbose_llvm_cpu_features: bool,
disable_c_depfile: bool,
is_test: bool,
+time_report: bool,
c_source_files: []const CSourceFile,
clang_argv: []const []const u8,
@@ -91,6 +93,10 @@ crt_files: std.StringHashMapUnmanaged(CRTFile) = .{},
/// Keeping track of this possibly open resource so we can close it later.
owned_link_dir: ?std.fs.Dir,
+/// This is for stage1 and should be deleted upon completion of self-hosting.
+/// Don't use this for anything other than stage1 compatibility.
+color: @import("main.zig").Color = .Auto,
+
pub const InnerError = Module.InnerError;
pub const CRTFile = struct {
@@ -643,100 +649,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
.ReleaseFast, .ReleaseSmall => false,
};
- const stage1_module: ?*stage1.Module = if (build_options.is_stage1 and use_llvm) blk: {
- // Here we use the legacy stage1 C++ compiler to compile Zig code.
- const stage2_target = try arena.create(stage1.Stage2Target);
- stage2_target.* = .{
- .arch = @enumToInt(options.target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
- .os = @enumToInt(options.target.os.tag),
- .abi = @enumToInt(options.target.abi),
- .is_native_os = options.is_native_os,
- .is_native_cpu = false, // Only true when bootstrapping the compiler.
- .llvm_cpu_name = if (options.target.cpu.model.llvm_name) |s| s.ptr else null,
- .llvm_cpu_features = llvm_cpu_features.?,
- };
- const progress = try arena.create(std.Progress);
- const main_progress_node = try progress.start("", 100);
- if (options.color == .Off) progress.terminal = null;
-
- const mod = module.?;
- const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{
- mod.root_pkg.root_src_path,
- });
- const zig_lib_dir = options.zig_lib_directory.path.?;
- const builtin_sub = &[_][]const u8{"builtin.zig"};
- const builtin_zig_path = try mod.zig_cache_artifact_directory.join(arena, builtin_sub);
-
- const stage1_module = stage1.create(
- @enumToInt(options.optimize_mode),
- undefined,
- 0, // TODO --main-pkg-path
- main_zig_file.ptr,
- main_zig_file.len,
- zig_lib_dir.ptr,
- zig_lib_dir.len,
- stage2_target,
- options.is_test,
- ) orelse return error.OutOfMemory;
-
- const output_dir = bin_directory.path orelse ".";
-
- const stage1_pkg = try arena.create(stage1.Pkg);
- stage1_pkg.* = .{
- .name_ptr = undefined,
- .name_len = 0,
- .path_ptr = undefined,
- .path_len = 0,
- .children_ptr = undefined,
- .children_len = 0,
- .parent = null,
- };
-
- stage1_module.* = .{
- .root_name_ptr = root_name.ptr,
- .root_name_len = root_name.len,
- .output_dir_ptr = output_dir.ptr,
- .output_dir_len = output_dir.len,
- .builtin_zig_path_ptr = builtin_zig_path.ptr,
- .builtin_zig_path_len = builtin_zig_path.len,
- .test_filter_ptr = "",
- .test_filter_len = 0,
- .test_name_prefix_ptr = "",
- .test_name_prefix_len = 0,
- .userdata = @ptrToInt(comp),
- .root_pkg = stage1_pkg,
- .code_model = @enumToInt(options.machine_code_model),
- .subsystem = stage1.TargetSubsystem.Auto,
- .err_color = @enumToInt(options.color),
- .pic = pic,
- .link_libc = options.link_libc,
- .link_libcpp = options.link_libcpp,
- .strip = options.strip,
- .is_single_threaded = single_threaded,
- .dll_export_fns = dll_export_fns,
- .link_mode_dynamic = link_mode == .Dynamic,
- .valgrind_enabled = valgrind,
- .function_sections = function_sections,
- .enable_stack_probing = stack_check,
- .enable_time_report = options.time_report,
- .enable_stack_report = false,
- .dump_analysis = false,
- .enable_doc_generation = false,
- .emit_bin = true,
- .emit_asm = false,
- .emit_llvm_ir = false,
- .test_is_evented = false,
- .verbose_tokenize = options.verbose_tokenize,
- .verbose_ast = options.verbose_ast,
- .verbose_ir = options.verbose_ir,
- .verbose_llvm_ir = options.verbose_llvm_ir,
- .verbose_cimport = options.verbose_cimport,
- .verbose_llvm_cpu_features = options.verbose_llvm_cpu_features,
- .main_progress_node = main_progress_node,
- };
- break :blk stage1_module;
- } else null;
-
const bin_file = try link.File.openPath(gpa, .{
.directory = bin_directory,
.sub_path = emit_bin.basename,
@@ -793,7 +705,6 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
.zig_lib_directory = options.zig_lib_directory,
.zig_cache_directory = options.zig_cache_directory,
.bin_file = bin_file,
- .stage1_module = stage1_module,
.work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa),
.keep_source_files_loaded = options.keep_source_files_loaded,
.use_clang = use_clang,
@@ -815,6 +726,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
.disable_c_depfile = options.disable_c_depfile,
.owned_link_dir = owned_link_dir,
.is_test = options.is_test,
+ .color = options.color,
+ .time_report = options.time_report,
};
break :comp comp;
};
@@ -845,7 +758,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
if (comp.wantBuildLibUnwindFromSource()) {
try comp.work_queue.writeItem(.{ .libunwind = {} });
}
- if (comp.stage1_module) |module| {
+ if (build_options.is_stage1 and comp.bin_file.options.use_llvm) {
try comp.work_queue.writeItem(.{ .stage1_module = {} });
}
if (is_exe_or_dyn_lib) {
@@ -858,15 +771,19 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
return comp;
}
+fn releaseStage1Lock(comp: *Compilation) void {
+ if (comp.stage1_lock) |*lock| {
+ lock.release();
+ comp.stage1_lock = null;
+ }
+}
+
pub fn destroy(self: *Compilation) void {
const optional_module = self.bin_file.options.module;
self.bin_file.destroy();
if (optional_module) |module| module.deinit();
- if (self.stage1_module) |module| {
- module.main_progress_node.?.end();
- module.destroy();
- }
+ self.releaseStage1Lock();
const gpa = self.gpa;
self.work_queue.deinit();
@@ -1200,8 +1117,9 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void {
};
},
.stage1_module => {
- // This Job is only queued up if there is a zig module.
- self.stage1_module.?.build_object();
+ self.updateStage1Module() catch |err| {
+ fatal("unable to build stage1 zig object: {}", .{@errorName(err)});
+ };
},
};
}
@@ -2174,3 +2092,155 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil
.lock = sub_compilation.bin_file.toOwnedLock(),
};
}
+
+fn updateStage1Module(comp: *Compilation) !void {
+ const tracy = trace(@src());
+ defer tracy.end();
+
+ var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
+ defer arena_allocator.deinit();
+ const arena = &arena_allocator.allocator;
+
+ // Here we use the legacy stage1 C++ compiler to compile Zig code.
+ const mod = comp.bin_file.options.module.?;
+ const directory = mod.zig_cache_artifact_directory; // Just an alias to make it shorter to type.
+ const main_zig_file = try mod.root_pkg.root_src_directory.join(arena, &[_][]const u8{
+ mod.root_pkg.root_src_path,
+ });
+ const zig_lib_dir = comp.zig_lib_directory.path.?;
+ const builtin_zig_path = try directory.join(arena, &[_][]const u8{"builtin.zig"});
+ const target = comp.getTarget();
+ const id_symlink_basename = "stage1.id";
+
+ // We are about to obtain this lock, so here we give other processes a chance first.
+ comp.releaseStage1Lock();
+
+ // Unlike with the self-hosted Zig module, stage1 does not support incremental compilation,
+ // so we input all the zig source files into the cache hash system. We're going to keep
+ // the artifact directory the same, however, so we take the same strategy as linking
+ // does where we have a file which specifies the hash of the output directory so that we can
+ // skip the expensive compilation step if the hash matches.
+ var ch = comp.cache_parent.obtain();
+ defer ch.deinit();
+
+ _ = try ch.addFile(main_zig_file, null);
+ ch.hash.add(comp.bin_file.options.valgrind);
+ ch.hash.add(comp.bin_file.options.single_threaded);
+ ch.hash.add(target.os.getVersionRange());
+ ch.hash.add(comp.bin_file.options.dll_export_fns);
+ ch.hash.add(comp.bin_file.options.function_sections);
+
+ if (try ch.hit()) {
+ const digest = ch.final();
+
+ var prev_digest_buf: [digest.len]u8 = undefined;
+ const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {
+ // Handle this as a cache miss.
+ break :blk prev_digest_buf[0..0];
+ };
+ if (mem.eql(u8, prev_digest, &digest)) {
+ comp.stage1_lock = ch.toOwnedLock();
+ return;
+ }
+ }
+
+ const stage2_target = try arena.create(stage1.Stage2Target);
+ stage2_target.* = .{
+ .arch = @enumToInt(target.cpu.arch) + 1, // skip over ZigLLVM_UnknownArch
+ .os = @enumToInt(target.os.tag),
+ .abi = @enumToInt(target.abi),
+ .is_native_os = comp.bin_file.options.is_native_os,
+ .is_native_cpu = false, // Only true when bootstrapping the compiler.
+ .llvm_cpu_name = if (target.cpu.model.llvm_name) |s| s.ptr else null,
+ .llvm_cpu_features = comp.bin_file.options.llvm_cpu_features.?,
+ };
+ var progress: std.Progress = .{};
+ var main_progress_node = try progress.start("", 100);
+ defer main_progress_node.end();
+ if (comp.color == .Off) progress.terminal = null;
+
+ comp.stage1_cache_hash = &ch;
+
+ const stage1_module = stage1.create(
+ @enumToInt(comp.bin_file.options.optimize_mode),
+ undefined,
+ 0, // TODO --main-pkg-path
+ main_zig_file.ptr,
+ main_zig_file.len,
+ zig_lib_dir.ptr,
+ zig_lib_dir.len,
+ stage2_target,
+ comp.is_test,
+ ) orelse return error.OutOfMemory;
+
+ const stage1_pkg = try arena.create(stage1.Pkg);
+ stage1_pkg.* = .{
+ .name_ptr = undefined,
+ .name_len = 0,
+ .path_ptr = undefined,
+ .path_len = 0,
+ .children_ptr = undefined,
+ .children_len = 0,
+ .parent = null,
+ };
+ const output_dir = comp.bin_file.options.directory.path orelse ".";
+ stage1_module.* = .{
+ .root_name_ptr = comp.bin_file.options.root_name.ptr,
+ .root_name_len = comp.bin_file.options.root_name.len,
+ .output_dir_ptr = output_dir.ptr,
+ .output_dir_len = output_dir.len,
+ .builtin_zig_path_ptr = builtin_zig_path.ptr,
+ .builtin_zig_path_len = builtin_zig_path.len,
+ .test_filter_ptr = "",
+ .test_filter_len = 0,
+ .test_name_prefix_ptr = "",
+ .test_name_prefix_len = 0,
+ .userdata = @ptrToInt(comp),
+ .root_pkg = stage1_pkg,
+ .code_model = @enumToInt(comp.bin_file.options.machine_code_model),
+ .subsystem = stage1.TargetSubsystem.Auto,
+ .err_color = @enumToInt(comp.color),
+ .pic = comp.bin_file.options.pic,
+ .link_libc = comp.bin_file.options.link_libc,
+ .link_libcpp = comp.bin_file.options.link_libcpp,
+ .strip = comp.bin_file.options.strip,
+ .is_single_threaded = comp.bin_file.options.single_threaded,
+ .dll_export_fns = comp.bin_file.options.dll_export_fns,
+ .link_mode_dynamic = comp.bin_file.options.link_mode == .Dynamic,
+ .valgrind_enabled = comp.bin_file.options.valgrind,
+ .function_sections = comp.bin_file.options.function_sections,
+ .enable_stack_probing = comp.bin_file.options.stack_check,
+ .enable_time_report = comp.time_report,
+ .enable_stack_report = false,
+ .dump_analysis = false,
+ .enable_doc_generation = false,
+ .emit_bin = true,
+ .emit_asm = false,
+ .emit_llvm_ir = false,
+ .test_is_evented = false,
+ .verbose_tokenize = comp.verbose_tokenize,
+ .verbose_ast = comp.verbose_ast,
+ .verbose_ir = comp.verbose_ir,
+ .verbose_llvm_ir = comp.verbose_llvm_ir,
+ .verbose_cimport = comp.verbose_cimport,
+ .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
+ .main_progress_node = main_progress_node,
+ };
+ stage1_module.build_object();
+ stage1_module.destroy();
+
+ const digest = ch.final();
+
+ // Update the dangling symlink with the digest. If it fails we can continue; it only
+ // means that the next invocation will have an unnecessary cache miss.
+ directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {
+ std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
+ };
+ // Again failure here only means an unnecessary cache miss.
+ ch.writeManifest() catch |err| {
+ std.log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
+ };
+ // We hang on to this lock so that the output file path can be used without
+ // other processes clobbering it.
+ comp.stage1_lock = ch.toOwnedLock();
+}
src-self-hosted/main.zig
@@ -1239,7 +1239,7 @@ pub fn buildOutputType(
fatal("TODO: implement `zig cc` when using it as a preprocessor", .{});
}
- if (build_options.is_stage1 and comp.stage1_module != null and watch) {
+ if (build_options.is_stage1 and comp.stage1_lock != null and watch) {
std.log.warn("--watch is not recommended with the stage1 backend; it leaks memory and is not capable of incremental compilation", .{});
}
src-self-hosted/stage1.zig
@@ -343,9 +343,9 @@ export fn stage2_fetch_file(
result_len: *usize,
) ?[*]const u8 {
const comp = @intToPtr(*Compilation, stage1.userdata);
- // TODO integrate this with cache hash
const file_path = path_ptr[0..path_len];
- const contents = std.fs.cwd().readFileAlloc(comp.gpa, file_path, std.math.maxInt(u32)) catch return null;
+ const max_file_size = std.math.maxInt(u32);
+ const contents = comp.stage1_cache_hash.addFilePostFetch(file_path, max_file_size) catch return null;
result_len.* = contents.len;
return contents.ptr;
}
BRANCH_TODO
@@ -1,6 +1,3 @@
- * Cache integration for stage1 zig code compilation
- * build & link against compiler-rt
- * build & link against freestanding libc
* resolve builtin.zig not working on windows & macos; try os_path_real
* build & link against libcxx and libcxxabi
* `zig test`