master
1const std = @import("std");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const Allocator = std.mem.Allocator;
5const log = std.log.scoped(.codegen);
6const math = std.math;
7const DW = std.dwarf;
8
9const Builder = std.zig.llvm.Builder;
10const llvm = if (build_options.have_llvm)
11 @import("llvm/bindings.zig")
12else
13 @compileError("LLVM unavailable");
14const link = @import("../link.zig");
15const Compilation = @import("../Compilation.zig");
16const build_options = @import("build_options");
17const Zcu = @import("../Zcu.zig");
18const InternPool = @import("../InternPool.zig");
19const Package = @import("../Package.zig");
20const Air = @import("../Air.zig");
21const Value = @import("../Value.zig");
22const Type = @import("../Type.zig");
23const codegen = @import("../codegen.zig");
24const x86_64_abi = @import("x86_64/abi.zig");
25const wasm_c_abi = @import("wasm/abi.zig");
26const aarch64_c_abi = @import("aarch64/abi.zig");
27const arm_c_abi = @import("arm/abi.zig");
28const riscv_c_abi = @import("riscv64/abi.zig");
29const mips_c_abi = @import("mips/abi.zig");
30const dev = @import("../dev.zig");
31
32const target_util = @import("../target.zig");
33const libcFloatPrefix = target_util.libcFloatPrefix;
34const libcFloatSuffix = target_util.libcFloatSuffix;
35const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
36const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
37
38const Error = error{ OutOfMemory, CodegenFail };
39
40pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
41 return comptime &.initMany(&.{
42 .expand_int_from_float_safe,
43 .expand_int_from_float_optimized_safe,
44 });
45}
46
47fn subArchName(target: *const std.Target, comptime family: std.Target.Cpu.Arch.Family, mappings: anytype) ?[]const u8 {
48 inline for (mappings) |mapping| {
49 if (target.cpu.has(family, mapping[0])) return mapping[1];
50 }
51
52 return null;
53}
54
55pub fn targetTriple(allocator: Allocator, target: *const std.Target) ![]const u8 {
56 var llvm_triple = std.array_list.Managed(u8).init(allocator);
57 defer llvm_triple.deinit();
58
59 const llvm_arch = switch (target.cpu.arch) {
60 .arm => "arm",
61 .armeb => "armeb",
62 .aarch64 => if (target.abi == .ilp32) "aarch64_32" else "aarch64",
63 .aarch64_be => "aarch64_be",
64 .arc => "arc",
65 .avr => "avr",
66 .bpfel => "bpfel",
67 .bpfeb => "bpfeb",
68 .csky => "csky",
69 .hexagon => "hexagon",
70 .loongarch32 => "loongarch32",
71 .loongarch64 => "loongarch64",
72 .m68k => "m68k",
73 // MIPS sub-architectures are a bit irregular, so we handle them manually here.
74 .mips => if (target.cpu.has(.mips, .mips32r6)) "mipsisa32r6" else "mips",
75 .mipsel => if (target.cpu.has(.mips, .mips32r6)) "mipsisa32r6el" else "mipsel",
76 .mips64 => if (target.cpu.has(.mips, .mips64r6)) "mipsisa64r6" else "mips64",
77 .mips64el => if (target.cpu.has(.mips, .mips64r6)) "mipsisa64r6el" else "mips64el",
78 .msp430 => "msp430",
79 .powerpc => "powerpc",
80 .powerpcle => "powerpcle",
81 .powerpc64 => "powerpc64",
82 .powerpc64le => "powerpc64le",
83 .amdgcn => "amdgcn",
84 .riscv32 => "riscv32",
85 .riscv32be => "riscv32be",
86 .riscv64 => "riscv64",
87 .riscv64be => "riscv64be",
88 .sparc => "sparc",
89 .sparc64 => "sparc64",
90 .s390x => "s390x",
91 .thumb => "thumb",
92 .thumbeb => "thumbeb",
93 .x86 => "i386",
94 .x86_64 => "x86_64",
95 .xcore => "xcore",
96 .xtensa => "xtensa",
97 .nvptx => "nvptx",
98 .nvptx64 => "nvptx64",
99 .spirv32 => switch (target.os.tag) {
100 .vulkan, .opengl => "spirv",
101 else => "spirv32",
102 },
103 .spirv64 => "spirv64",
104 .lanai => "lanai",
105 .wasm32 => "wasm32",
106 .wasm64 => "wasm64",
107 .ve => "ve",
108
109 .alpha,
110 .arceb,
111 .hppa,
112 .hppa64,
113 .kalimba,
114 .kvx,
115 .microblaze,
116 .microblazeel,
117 .or1k,
118 .propeller,
119 .sh,
120 .sheb,
121 .x86_16,
122 .xtensaeb,
123 => unreachable, // Gated by hasLlvmSupport().
124 };
125
126 try llvm_triple.appendSlice(llvm_arch);
127
128 const llvm_sub_arch: ?[]const u8 = switch (target.cpu.arch) {
129 .arm, .armeb, .thumb, .thumbeb => subArchName(target, .arm, .{
130 .{ .v4t, "v4t" },
131 .{ .v5t, "v5t" },
132 .{ .v5te, "v5te" },
133 .{ .v5tej, "v5tej" },
134 .{ .v6, "v6" },
135 .{ .v6k, "v6k" },
136 .{ .v6kz, "v6kz" },
137 .{ .v6m, "v6m" },
138 .{ .v6t2, "v6t2" },
139 .{ .v7a, "v7a" },
140 .{ .v7em, "v7em" },
141 .{ .v7m, "v7m" },
142 .{ .v7r, "v7r" },
143 .{ .v7ve, "v7ve" },
144 .{ .v8a, "v8a" },
145 .{ .v8_1a, "v8.1a" },
146 .{ .v8_2a, "v8.2a" },
147 .{ .v8_3a, "v8.3a" },
148 .{ .v8_4a, "v8.4a" },
149 .{ .v8_5a, "v8.5a" },
150 .{ .v8_6a, "v8.6a" },
151 .{ .v8_7a, "v8.7a" },
152 .{ .v8_8a, "v8.8a" },
153 .{ .v8_9a, "v8.9a" },
154 .{ .v8m, "v8m.base" },
155 .{ .v8m_main, "v8m.main" },
156 .{ .v8_1m_main, "v8.1m.main" },
157 .{ .v8r, "v8r" },
158 .{ .v9a, "v9a" },
159 .{ .v9_1a, "v9.1a" },
160 .{ .v9_2a, "v9.2a" },
161 .{ .v9_3a, "v9.3a" },
162 .{ .v9_4a, "v9.4a" },
163 .{ .v9_5a, "v9.5a" },
164 .{ .v9_6a, "v9.6a" },
165 }),
166 .powerpc => subArchName(target, .powerpc, .{
167 .{ .spe, "spe" },
168 }),
169 .spirv32, .spirv64 => subArchName(target, .spirv, .{
170 .{ .v1_6, "1.6" },
171 .{ .v1_5, "1.5" },
172 .{ .v1_4, "1.4" },
173 .{ .v1_3, "1.3" },
174 .{ .v1_2, "1.2" },
175 .{ .v1_1, "1.1" },
176 }),
177 else => null,
178 };
179
180 if (llvm_sub_arch) |sub| try llvm_triple.appendSlice(sub);
181 try llvm_triple.append('-');
182
183 try llvm_triple.appendSlice(switch (target.os.tag) {
184 .driverkit,
185 .ios,
186 .maccatalyst,
187 .macos,
188 .tvos,
189 .visionos,
190 .watchos,
191 => "apple",
192 .ps4,
193 .ps5,
194 => "scei",
195 .amdhsa,
196 .amdpal,
197 => "amd",
198 .cuda,
199 .nvcl,
200 => "nvidia",
201 .mesa3d,
202 => "mesa",
203 else => "unknown",
204 });
205 try llvm_triple.append('-');
206
207 const llvm_os = switch (target.os.tag) {
208 .dragonfly => "dragonfly",
209 .freebsd => "freebsd",
210 .fuchsia => "fuchsia",
211 .linux => "linux",
212 .netbsd => "netbsd",
213 .openbsd => "openbsd",
214 .illumos => "solaris",
215 .windows, .uefi => "windows",
216 .haiku => "haiku",
217 .rtems => "rtems",
218 .cuda => "cuda",
219 .nvcl => "nvcl",
220 .amdhsa => "amdhsa",
221 .ps3 => "lv2",
222 .ps4 => "ps4",
223 .ps5 => "ps5",
224 .mesa3d => "mesa3d",
225 .amdpal => "amdpal",
226 .hermit => "hermit",
227 .hurd => "hurd",
228 .wasi => "wasi",
229 .emscripten => "emscripten",
230 .macos => "macosx",
231 .ios, .maccatalyst => "ios",
232 .tvos => "tvos",
233 .watchos => "watchos",
234 .driverkit => "driverkit",
235 .visionos => "xros",
236 .serenity => "serenity",
237 .vulkan => "vulkan",
238 .managarm => "managarm",
239
240 .@"3ds",
241 .contiki,
242 .freestanding,
243 .opencl, // https://llvm.org/docs/SPIRVUsage.html#target-triples
244 .opengl,
245 .other,
246 .plan9,
247 .vita,
248 => "unknown",
249 };
250 try llvm_triple.appendSlice(llvm_os);
251
252 switch (target.os.versionRange()) {
253 .none,
254 .windows,
255 => {},
256 .semver => |ver| try llvm_triple.print("{d}.{d}.{d}", .{
257 ver.min.major,
258 ver.min.minor,
259 ver.min.patch,
260 }),
261 inline .linux, .hurd => |ver| try llvm_triple.print("{d}.{d}.{d}", .{
262 ver.range.min.major,
263 ver.range.min.minor,
264 ver.range.min.patch,
265 }),
266 }
267 try llvm_triple.append('-');
268
269 const llvm_abi = switch (target.abi) {
270 .none => if (target.os.tag == .maccatalyst) "macabi" else "unknown",
271 .gnu => "gnu",
272 .gnuabin32 => "gnuabin32",
273 .gnuabi64 => "gnuabi64",
274 .gnueabi => "gnueabi",
275 .gnueabihf => "gnueabihf",
276 .gnuf32 => "gnuf32",
277 .gnusf => "gnusf",
278 .gnux32 => "gnux32",
279 .ilp32 => "unknown",
280 .eabi => "eabi",
281 .eabihf => "eabihf",
282 .android => "android",
283 .androideabi => "androideabi",
284 .musl => switch (target.os.tag) {
285 // For WASI/Emscripten, "musl" refers to the libc, not really the ABI.
286 // "unknown" provides better compatibility with LLVM-based tooling for these targets.
287 .wasi, .emscripten => "unknown",
288 else => "musl",
289 },
290 .muslabin32 => "muslabin32",
291 .muslabi64 => "muslabi64",
292 .musleabi => "musleabi",
293 .musleabihf => "musleabihf",
294 .muslf32 => "muslf32",
295 .muslsf => "muslsf",
296 .muslx32 => "muslx32",
297 .msvc => "msvc",
298 .itanium => "itanium",
299 .simulator => "simulator",
300 .ohos, .ohoseabi => "ohos",
301 };
302 try llvm_triple.appendSlice(llvm_abi);
303
304 switch (target.os.versionRange()) {
305 .none,
306 .semver,
307 .windows,
308 => {},
309 inline .hurd, .linux => |ver| if (target.abi.isGnu()) {
310 try llvm_triple.print("{d}.{d}.{d}", .{
311 ver.glibc.major,
312 ver.glibc.minor,
313 ver.glibc.patch,
314 });
315 } else if (@TypeOf(ver) == std.Target.Os.LinuxVersionRange and target.abi.isAndroid()) {
316 try llvm_triple.print("{d}", .{ver.android});
317 },
318 }
319
320 return llvm_triple.toOwnedSlice();
321}
322
323pub fn supportsTailCall(target: *const std.Target) bool {
324 return switch (target.cpu.arch) {
325 .wasm32, .wasm64 => target.cpu.has(.wasm, .tail_call),
326 // Although these ISAs support tail calls, LLVM does not support tail calls on them.
327 .mips, .mipsel, .mips64, .mips64el => false,
328 .powerpc, .powerpcle, .powerpc64, .powerpc64le => false,
329 else => true,
330 };
331}
332
333pub fn dataLayout(target: *const std.Target) []const u8 {
334 // These data layouts should match Clang.
335 return switch (target.cpu.arch) {
336 .arc => "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-f32:32:32-i64:32-f64:32-a:0:32-n32",
337 .xcore => "e-m:e-p:32:32-i1:8:32-i8:8:32-i16:16:32-i64:32-f64:32-a:0:32-n32",
338 .hexagon => "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048",
339 .lanai => "E-m:e-p:32:32-i64:64-a:0:32-n32-S64",
340 .aarch64 => if (target.ofmt == .macho)
341 if (target.os.tag == .windows or target.os.tag == .uefi)
342 "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
343 else if (target.abi == .ilp32)
344 "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
345 else
346 "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
347 else if (target.os.tag == .windows or target.os.tag == .uefi)
348 "e-m:w-p270:32:32-p271:32:32-p272:64:64-p:64:64-i32:32-i64:64-i128:128-n32:64-S128-Fn32"
349 else
350 "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
351 .aarch64_be => "E-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
352 .arm => if (target.ofmt == .macho)
353 "e-m:o-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
354 else
355 "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64",
356 .armeb, .thumbeb => if (target.ofmt == .macho)
357 "E-m:o-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
358 else
359 "E-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64",
360 .thumb => if (target.ofmt == .macho)
361 "e-m:o-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
362 else if (target.os.tag == .windows or target.os.tag == .uefi)
363 "e-m:w-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
364 else
365 "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64",
366 .avr => "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8",
367 .bpfeb => "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128",
368 .bpfel => "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128",
369 .msp430 => "e-m:e-p:16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16",
370 .mips => "E-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64",
371 .mipsel => "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64",
372 .mips64 => switch (target.abi) {
373 .gnuabin32, .muslabin32 => "E-m:e-p:32:32-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128",
374 else => "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128",
375 },
376 .mips64el => switch (target.abi) {
377 .gnuabin32, .muslabin32 => "e-m:e-p:32:32-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128",
378 else => "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128",
379 },
380 .m68k => "E-m:e-p:32:16:32-i8:8:8-i16:16:16-i32:16:32-n8:16:32-a:0:16-S16",
381 .powerpc => "E-m:e-p:32:32-Fn32-i64:64-n32",
382 .powerpcle => "e-m:e-p:32:32-Fn32-i64:64-n32",
383 .powerpc64 => switch (target.os.tag) {
384 .linux => if (target.abi.isMusl())
385 "E-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
386 else
387 "E-m:e-Fi64-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512",
388 .ps3 => "E-m:e-p:32:32-Fi64-i64:64-i128:128-n32:64",
389 else => if (target.os.tag == .openbsd or
390 (target.os.tag == .freebsd and target.os.version_range.semver.isAtLeast(.{ .major = 13, .minor = 0, .patch = 0 }) orelse false))
391 "E-m:e-Fn32-i64:64-i128:128-n32:64"
392 else
393 "E-m:e-Fi64-i64:64-i128:128-n32:64",
394 },
395 .powerpc64le => if (target.os.tag == .linux)
396 "e-m:e-Fn32-i64:64-i128:128-n32:64-S128-v256:256:256-v512:512:512"
397 else
398 "e-m:e-Fn32-i64:64-i128:128-n32:64",
399 .nvptx => "e-p:32:32-p6:32:32-p7:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64",
400 .nvptx64 => "e-p6:32:32-i64:64-i128:128-v16:16-v32:32-n16:32:64",
401 .amdgcn => "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9",
402 .riscv32 => if (target.cpu.has(.riscv, .e))
403 "e-m:e-p:32:32-i64:64-n32-S32"
404 else
405 "e-m:e-p:32:32-i64:64-n32-S128",
406 .riscv32be => if (target.cpu.has(.riscv, .e))
407 "E-m:e-p:32:32-i64:64-n32-S32"
408 else
409 "E-m:e-p:32:32-i64:64-n32-S128",
410 .riscv64 => if (target.cpu.has(.riscv, .e))
411 "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64"
412 else
413 "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128",
414 .riscv64be => if (target.cpu.has(.riscv, .e))
415 "E-m:e-p:64:64-i64:64-i128:128-n32:64-S64"
416 else
417 "E-m:e-p:64:64-i64:64-i128:128-n32:64-S128",
418 .sparc => "E-m:e-p:32:32-i64:64-i128:128-f128:64-n32-S64",
419 .sparc64 => "E-m:e-i64:64-i128:128-n32:64-S128",
420 .s390x => "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64",
421 .x86 => if (target.os.tag == .windows or target.os.tag == .uefi) switch (target.abi) {
422 .gnu => if (target.ofmt == .coff)
423 "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
424 else
425 "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32",
426 else => blk: {
427 const msvc = switch (target.abi) {
428 .none, .msvc => true,
429 else => false,
430 };
431
432 break :blk if (target.ofmt == .coff)
433 if (msvc)
434 "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
435 else
436 "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32"
437 else if (msvc)
438 "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32-a:0:32-S32"
439 else
440 "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:32-n8:16:32-a:0:32-S32";
441 },
442 } else if (target.ofmt == .macho)
443 "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128"
444 else
445 "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i128:128-f64:32:64-f80:32-n8:16:32-S128",
446 .x86_64 => if (target.os.tag.isDarwin() or target.ofmt == .macho)
447 "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
448 else switch (target.abi) {
449 .gnux32, .muslx32 => "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128",
450 else => if ((target.os.tag == .windows or target.os.tag == .uefi) and target.ofmt == .coff)
451 "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
452 else
453 "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128",
454 },
455 .spirv32 => switch (target.os.tag) {
456 .vulkan, .opengl => "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1",
457 else => "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1",
458 },
459 .spirv64 => "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1",
460 .wasm32 => if (target.os.tag == .emscripten)
461 "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-S128-ni:1:10:20"
462 else
463 "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20",
464 .wasm64 => if (target.os.tag == .emscripten)
465 "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-f128:64-n32:64-S128-ni:1:10:20"
466 else
467 "e-m:e-p:64:64-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20",
468 .ve => "e-m:e-i64:64-n32:64-S128-v64:64:64-v128:64:64-v256:64:64-v512:64:64-v1024:64:64-v2048:64:64-v4096:64:64-v8192:64:64-v16384:64:64",
469 .csky => "e-m:e-S32-p:32:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:32-v128:32:32-a:0:32-Fi32-n32",
470 .loongarch32 => "e-m:e-p:32:32-i64:64-n32-S128",
471 .loongarch64 => "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128",
472 .xtensa => "e-m:e-p:32:32-i8:8:32-i16:16:32-i64:64-n32",
473
474 .alpha,
475 .arceb,
476 .hppa,
477 .hppa64,
478 .kalimba,
479 .kvx,
480 .microblaze,
481 .microblazeel,
482 .or1k,
483 .propeller,
484 .sh,
485 .sheb,
486 .x86_16,
487 .xtensaeb,
488 => unreachable, // Gated by hasLlvmSupport().
489 };
490}
491
492// Avoid depending on `llvm.CodeModel` in the bitcode-only case.
493const CodeModel = enum {
494 default,
495 tiny,
496 small,
497 kernel,
498 medium,
499 large,
500};
501
502fn codeModel(model: std.builtin.CodeModel, target: *const std.Target) CodeModel {
503 // Roughly match Clang's mapping of GCC code models to LLVM code models.
504 return switch (model) {
505 .default => .default,
506 .extreme, .large => .large,
507 .kernel => .kernel,
508 .medany => if (target.cpu.arch.isRISCV()) .medium else .large,
509 .medium => .medium,
510 .medmid => .medium,
511 .normal, .medlow, .small => .small,
512 .tiny => .tiny,
513 };
514}
515
516pub const Object = struct {
517 gpa: Allocator,
518 builder: Builder,
519
520 debug_compile_unit: Builder.Metadata.Optional,
521
522 debug_enums_fwd_ref: Builder.Metadata.Optional,
523 debug_globals_fwd_ref: Builder.Metadata.Optional,
524
525 debug_enums: std.ArrayList(Builder.Metadata),
526 debug_globals: std.ArrayList(Builder.Metadata),
527
528 debug_file_map: std.AutoHashMapUnmanaged(Zcu.File.Index, Builder.Metadata),
529 debug_type_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Metadata),
530
531 debug_unresolved_namespace_scopes: std.AutoArrayHashMapUnmanaged(InternPool.NamespaceIndex, Builder.Metadata),
532
533 target: *const std.Target,
534 /// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function,
535 /// but that has some downsides:
536 /// * we have to compute the fully qualified name every time we want to do the lookup
537 /// * for externally linked functions, the name is not fully qualified, but when
538 /// a Decl goes from exported to not exported and vice-versa, we would use the wrong
539 /// version of the name and incorrectly get function not found in the llvm module.
540 /// * it works for functions not all globals.
541 /// Therefore, this table keeps track of the mapping.
542 nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index),
543 /// Same deal as `decl_map` but for anonymous declarations, which are always global constants.
544 uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index),
545 /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction.
546 enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index),
547 /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction.
548 named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index),
549 /// Maps Zig types to LLVM types. The table memory is backed by the GPA of
550 /// the compiler.
551 /// TODO when InternPool garbage collection is implemented, this map needs
552 /// to be garbage collected as well.
553 type_map: TypeMap,
554 /// The LLVM global table which holds the names corresponding to Zig errors.
555 /// Note that the values are not added until `emit`, when all errors in
556 /// the compilation are known.
557 error_name_table: Builder.Variable.Index,
558
559 /// Memoizes a null `?usize` value.
560 null_opt_usize: Builder.Constant,
561
562 /// When an LLVM struct type is created, an entry is inserted into this
563 /// table for every zig source field of the struct that has a corresponding
564 /// LLVM struct field. comptime fields are not included. Zero-bit fields are
565 /// mapped to a field at the correct byte, which may be a padding field, or
566 /// are not mapped, in which case they are semantically at the end of the
567 /// struct.
568 /// The value is the LLVM struct field index.
569 /// This is denormalized data.
570 struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
571
572 /// Values for `@llvm.used`.
573 used: std.ArrayList(Builder.Constant),
574
575 const ZigStructField = struct {
576 struct_ty: InternPool.Index,
577 field_index: u32,
578 };
579
580 pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
581
582 pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
583
584 pub fn create(arena: Allocator, comp: *Compilation) !Ptr {
585 dev.check(.llvm_backend);
586 const gpa = comp.gpa;
587 const target = &comp.root_mod.resolved_target.result;
588 const llvm_target_triple = try targetTriple(arena, target);
589
590 var builder = try Builder.init(.{
591 .allocator = gpa,
592 .strip = comp.config.debug_format == .strip,
593 .name = comp.root_name,
594 .target = target,
595 .triple = llvm_target_triple,
596 });
597 errdefer builder.deinit();
598
599 builder.data_layout = try builder.string(dataLayout(target));
600
601 const debug_compile_unit, const debug_enums_fwd_ref, const debug_globals_fwd_ref =
602 if (!builder.strip) debug_info: {
603 // We fully resolve all paths at this point to avoid lack of
604 // source line info in stack traces or lack of debugging
605 // information which, if relative paths were used, would be
606 // very location dependent.
607 // TODO: the only concern I have with this is WASI as either host or target, should
608 // we leave the paths as relative then?
609 // TODO: This is totally wrong. In dwarf, paths are encoded as relative to
610 // a particular directory, and then the directory path is specified elsewhere.
611 // In the compiler frontend we have it stored correctly in this
612 // way already, but here we throw all that sweet information
613 // into the garbage can by converting into absolute paths. What
614 // a terrible tragedy.
615 const compile_unit_dir = blk: {
616 const zcu = comp.zcu orelse break :blk comp.dirs.cwd;
617 break :blk try zcu.main_mod.root.toAbsolute(comp.dirs, arena);
618 };
619
620 const debug_file = try builder.debugFile(
621 try builder.metadataString(comp.root_name),
622 try builder.metadataString(compile_unit_dir),
623 );
624
625 const debug_enums_fwd_ref = try builder.debugForwardReference();
626 const debug_globals_fwd_ref = try builder.debugForwardReference();
627
628 const debug_compile_unit = try builder.debugCompileUnit(
629 debug_file,
630 // Don't use the version string here; LLVM misparses it when it
631 // includes the git revision.
632 try builder.metadataStringFmt("zig {d}.{d}.{d}", .{
633 build_options.semver.major,
634 build_options.semver.minor,
635 build_options.semver.patch,
636 }),
637 debug_enums_fwd_ref,
638 debug_globals_fwd_ref,
639 .{ .optimized = comp.root_mod.optimize_mode != .Debug },
640 );
641
642 try builder.addNamedMetadata(try builder.string("llvm.dbg.cu"), &.{debug_compile_unit});
643 break :debug_info .{
644 debug_compile_unit.toOptional(),
645 debug_enums_fwd_ref.toOptional(),
646 debug_globals_fwd_ref.toOptional(),
647 };
648 } else .{Builder.Metadata.Optional.none} ** 3;
649
650 const obj = try arena.create(Object);
651 obj.* = .{
652 .gpa = gpa,
653 .builder = builder,
654 .debug_compile_unit = debug_compile_unit,
655 .debug_enums_fwd_ref = debug_enums_fwd_ref,
656 .debug_globals_fwd_ref = debug_globals_fwd_ref,
657 .debug_enums = .{},
658 .debug_globals = .{},
659 .debug_file_map = .{},
660 .debug_type_map = .{},
661 .debug_unresolved_namespace_scopes = .{},
662 .target = target,
663 .nav_map = .{},
664 .uav_map = .{},
665 .enum_tag_name_map = .{},
666 .named_enum_map = .{},
667 .type_map = .{},
668 .error_name_table = .none,
669 .null_opt_usize = .no_init,
670 .struct_field_map = .{},
671 .used = .{},
672 };
673 return obj;
674 }
675
676 pub fn deinit(self: *Object) void {
677 const gpa = self.gpa;
678 self.debug_enums.deinit(gpa);
679 self.debug_globals.deinit(gpa);
680 self.debug_file_map.deinit(gpa);
681 self.debug_type_map.deinit(gpa);
682 self.debug_unresolved_namespace_scopes.deinit(gpa);
683 self.nav_map.deinit(gpa);
684 self.uav_map.deinit(gpa);
685 self.enum_tag_name_map.deinit(gpa);
686 self.named_enum_map.deinit(gpa);
687 self.type_map.deinit(gpa);
688 self.builder.deinit();
689 self.struct_field_map.deinit(gpa);
690 self.* = undefined;
691 }
692
693 fn genErrorNameTable(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {
694 // If o.error_name_table is null, then it was not referenced by any instructions.
695 if (o.error_name_table == .none) return;
696
697 const zcu = pt.zcu;
698 const ip = &zcu.intern_pool;
699
700 const error_name_list = ip.global_error_set.getNamesFromMainThread();
701 const llvm_errors = try zcu.gpa.alloc(Builder.Constant, 1 + error_name_list.len);
702 defer zcu.gpa.free(llvm_errors);
703
704 // TODO: Address space
705 const slice_ty = Type.slice_const_u8_sentinel_0;
706 const llvm_usize_ty = try o.lowerType(pt, Type.usize);
707 const llvm_slice_ty = try o.lowerType(pt, slice_ty);
708 const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
709
710 llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
711 for (llvm_errors[1..], error_name_list) |*llvm_error, name| {
712 const name_string = try o.builder.stringNull(name.toSlice(ip));
713 const name_init = try o.builder.stringConst(name_string);
714 const name_variable_index =
715 try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
716 try name_variable_index.setInitializer(name_init, &o.builder);
717 name_variable_index.setLinkage(.private, &o.builder);
718 name_variable_index.setMutability(.constant, &o.builder);
719 name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
720 name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
721
722 llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{
723 name_variable_index.toConst(&o.builder),
724 try o.builder.intConst(llvm_usize_ty, name_string.slice(&o.builder).?.len - 1),
725 });
726 }
727
728 const table_variable_index = try o.builder.addVariable(.empty, llvm_table_ty, .default);
729 try table_variable_index.setInitializer(
730 try o.builder.arrayConst(llvm_table_ty, llvm_errors),
731 &o.builder,
732 );
733 table_variable_index.setLinkage(.private, &o.builder);
734 table_variable_index.setMutability(.constant, &o.builder);
735 table_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
736 table_variable_index.setAlignment(
737 slice_ty.abiAlignment(zcu).toLlvm(),
738 &o.builder,
739 );
740
741 try o.error_name_table.setInitializer(table_variable_index.toConst(&o.builder), &o.builder);
742 }
743
744 fn genCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !void {
745 // If there is no such function in the module, it means the source code does not need it.
746 const name = o.builder.strtabStringIfExists(lt_errors_fn_name) orelse return;
747 const llvm_fn = o.builder.getGlobal(name) orelse return;
748 const errors_len = pt.zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
749
750 var wip = try Builder.WipFunction.init(&o.builder, .{
751 .function = llvm_fn.ptrConst(&o.builder).kind.function,
752 .strip = true,
753 });
754 defer wip.deinit();
755 wip.cursor = .{ .block = try wip.block(0, "Entry") };
756
757 // Example source of the following LLVM IR:
758 // fn __zig_lt_errors_len(index: u16) bool {
759 // return index <= total_errors_len;
760 // }
761
762 const lhs = wip.arg(0);
763 const rhs = try o.builder.intValue(try o.errorIntType(pt), errors_len);
764 const is_lt = try wip.icmp(.ule, lhs, rhs, "");
765 _ = try wip.ret(is_lt);
766 try wip.finish();
767 }
768
769 fn genModuleLevelAssembly(object: *Object, pt: Zcu.PerThread) Allocator.Error!void {
770 const b = &object.builder;
771 const gpa = b.gpa;
772 b.module_asm.clearRetainingCapacity();
773 for (pt.zcu.global_assembly.values()) |assembly| {
774 try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1);
775 b.module_asm.appendSliceAssumeCapacity(assembly);
776 b.module_asm.appendAssumeCapacity('\n');
777 }
778 if (b.module_asm.getLastOrNull()) |last| {
779 if (last != '\n') try b.module_asm.append(gpa, '\n');
780 }
781 }
782
783 pub const EmitOptions = struct {
784 pre_ir_path: ?[]const u8,
785 pre_bc_path: ?[]const u8,
786 bin_path: ?[:0]const u8,
787 asm_path: ?[:0]const u8,
788 post_ir_path: ?[:0]const u8,
789 post_bc_path: ?[]const u8,
790
791 is_debug: bool,
792 is_small: bool,
793 time_report: ?*Compilation.TimeReport,
794 sanitize_thread: bool,
795 fuzz: bool,
796 lto: std.zig.LtoMode,
797 };
798
799 pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ LinkFailure, OutOfMemory }!void {
800 const zcu = pt.zcu;
801 const comp = zcu.comp;
802 const diags = &comp.link_diags;
803
804 {
805 try o.genErrorNameTable(pt);
806 try o.genCmpLtErrorsLenFunction(pt);
807 try o.genModuleLevelAssembly(pt);
808
809 if (o.used.items.len > 0) {
810 const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr);
811 const init_val = try o.builder.arrayConst(array_llvm_ty, o.used.items);
812 const compiler_used_variable = try o.builder.addVariable(
813 try o.builder.strtabString("llvm.used"),
814 array_llvm_ty,
815 .default,
816 );
817 compiler_used_variable.setLinkage(.appending, &o.builder);
818 compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder);
819 try compiler_used_variable.setInitializer(init_val, &o.builder);
820 }
821
822 if (!o.builder.strip) {
823 {
824 var i: usize = 0;
825 while (i < o.debug_unresolved_namespace_scopes.count()) : (i += 1) {
826 const namespace_index = o.debug_unresolved_namespace_scopes.keys()[i];
827 const fwd_ref = o.debug_unresolved_namespace_scopes.values()[i];
828
829 const namespace = zcu.namespacePtr(namespace_index);
830 const debug_type = try o.lowerDebugType(pt, Type.fromInterned(namespace.owner_type));
831
832 o.builder.resolveDebugForwardReference(fwd_ref, debug_type);
833 }
834 }
835
836 o.builder.resolveDebugForwardReference(
837 o.debug_enums_fwd_ref.unwrap().?,
838 try o.builder.metadataTuple(o.debug_enums.items),
839 );
840
841 o.builder.resolveDebugForwardReference(
842 o.debug_globals_fwd_ref.unwrap().?,
843 try o.builder.metadataTuple(o.debug_globals.items),
844 );
845 }
846 }
847
848 {
849 var module_flags = try std.array_list.Managed(Builder.Metadata).initCapacity(o.gpa, 8);
850 defer module_flags.deinit();
851
852 const behavior_error = try o.builder.metadataConstant(try o.builder.intConst(.i32, 1));
853 const behavior_warning = try o.builder.metadataConstant(try o.builder.intConst(.i32, 2));
854 const behavior_max = try o.builder.metadataConstant(try o.builder.intConst(.i32, 7));
855 const behavior_min = try o.builder.metadataConstant(try o.builder.intConst(.i32, 8));
856
857 if (target_util.llvmMachineAbi(&comp.root_mod.resolved_target.result)) |abi| {
858 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
859 behavior_error,
860 (try o.builder.metadataString("target-abi")).toMetadata(),
861 (try o.builder.metadataString(abi)).toMetadata(),
862 }));
863 }
864
865 const pic_level = target_util.picLevel(&comp.root_mod.resolved_target.result);
866 if (comp.root_mod.pic) {
867 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
868 behavior_min,
869 (try o.builder.metadataString("PIC Level")).toMetadata(),
870 try o.builder.metadataConstant(try o.builder.intConst(.i32, pic_level)),
871 }));
872 }
873
874 if (comp.config.pie) {
875 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
876 behavior_max,
877 (try o.builder.metadataString("PIE Level")).toMetadata(),
878 try o.builder.metadataConstant(try o.builder.intConst(.i32, pic_level)),
879 }));
880 }
881
882 if (comp.root_mod.code_model != .default) {
883 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
884 behavior_error,
885 (try o.builder.metadataString("Code Model")).toMetadata(),
886 try o.builder.metadataConstant(try o.builder.intConst(.i32, @as(
887 i32,
888 switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
889 .default => unreachable,
890 .tiny => 0,
891 .small => 1,
892 .kernel => 2,
893 .medium => 3,
894 .large => 4,
895 },
896 ))),
897 }));
898 }
899
900 if (!o.builder.strip) {
901 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
902 behavior_warning,
903 (try o.builder.metadataString("Debug Info Version")).toMetadata(),
904 try o.builder.metadataConstant(try o.builder.intConst(.i32, 3)),
905 }));
906
907 switch (comp.config.debug_format) {
908 .strip => unreachable,
909 .dwarf => |f| {
910 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
911 behavior_max,
912 (try o.builder.metadataString("Dwarf Version")).toMetadata(),
913 try o.builder.metadataConstant(try o.builder.intConst(.i32, 4)),
914 }));
915
916 if (f == .@"64") {
917 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
918 behavior_max,
919 (try o.builder.metadataString("DWARF64")).toMetadata(),
920 try o.builder.metadataConstant(.@"1"),
921 }));
922 }
923 },
924 .code_view => {
925 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
926 behavior_warning,
927 (try o.builder.metadataString("CodeView")).toMetadata(),
928 try o.builder.metadataConstant(.@"1"),
929 }));
930 },
931 }
932 }
933
934 const target = &comp.root_mod.resolved_target.result;
935 if (target.os.tag == .windows and (target.cpu.arch == .x86_64 or target.cpu.arch == .x86)) {
936 // Add the "RegCallv4" flag so that any functions using `x86_regcallcc` use regcall
937 // v4, which is essentially a requirement on Windows. See corresponding logic in
938 // `toLlvmCallConvTag`.
939 module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
940 behavior_max,
941 (try o.builder.metadataString("RegCallv4")).toMetadata(),
942 try o.builder.metadataConstant(.@"1"),
943 }));
944 }
945
946 try o.builder.addNamedMetadata(try o.builder.string("llvm.module.flags"), module_flags.items);
947 }
948
949 const target_triple_sentinel =
950 try o.gpa.dupeZ(u8, o.builder.target_triple.slice(&o.builder).?);
951 defer o.gpa.free(target_triple_sentinel);
952
953 const emit_asm_msg = options.asm_path orelse "(none)";
954 const emit_bin_msg = options.bin_path orelse "(none)";
955 const post_llvm_ir_msg = options.post_ir_path orelse "(none)";
956 const post_llvm_bc_msg = options.post_bc_path orelse "(none)";
957 log.debug("emit LLVM object asm={s} bin={s} ir={s} bc={s}", .{
958 emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg,
959 });
960
961 const context, const module = emit: {
962 if (options.pre_ir_path) |path| {
963 if (std.mem.eql(u8, path, "-")) {
964 o.builder.dump();
965 } else {
966 o.builder.printToFilePath(std.fs.cwd(), path) catch |err| {
967 log.err("failed printing LLVM module to \"{s}\": {s}", .{ path, @errorName(err) });
968 };
969 }
970 }
971
972 const bitcode = try o.builder.toBitcode(o.gpa, .{
973 .name = "zig",
974 .version = build_options.semver,
975 });
976 defer o.gpa.free(bitcode);
977 o.builder.clearAndFree();
978
979 if (options.pre_bc_path) |path| {
980 var file = std.fs.cwd().createFile(path, .{}) catch |err|
981 return diags.fail("failed to create '{s}': {s}", .{ path, @errorName(err) });
982 defer file.close();
983
984 const ptr: [*]const u8 = @ptrCast(bitcode.ptr);
985 file.writeAll(ptr[0..(bitcode.len * 4)]) catch |err|
986 return diags.fail("failed to write to '{s}': {s}", .{ path, @errorName(err) });
987 }
988
989 if (options.asm_path == null and options.bin_path == null and
990 options.post_ir_path == null and options.post_bc_path == null) return;
991
992 if (options.post_bc_path) |path| {
993 var file = std.fs.cwd().createFile(path, .{}) catch |err|
994 return diags.fail("failed to create '{s}': {s}", .{ path, @errorName(err) });
995 defer file.close();
996
997 const ptr: [*]const u8 = @ptrCast(bitcode.ptr);
998 file.writeAll(ptr[0..(bitcode.len * 4)]) catch |err|
999 return diags.fail("failed to write to '{s}': {s}", .{ path, @errorName(err) });
1000 }
1001
1002 if (!build_options.have_llvm or !comp.config.use_lib_llvm) {
1003 return diags.fail("emitting without libllvm not implemented", .{});
1004 }
1005
1006 initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch);
1007
1008 const context: *llvm.Context = llvm.Context.create();
1009 errdefer context.dispose();
1010
1011 const bitcode_memory_buffer = llvm.MemoryBuffer.createMemoryBufferWithMemoryRange(
1012 @ptrCast(bitcode.ptr),
1013 bitcode.len * 4,
1014 "BitcodeBuffer",
1015 llvm.Bool.False,
1016 );
1017 defer bitcode_memory_buffer.dispose();
1018
1019 context.enableBrokenDebugInfoCheck();
1020
1021 var module: *llvm.Module = undefined;
1022 if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) {
1023 return diags.fail("Failed to parse bitcode", .{});
1024 }
1025 break :emit .{ context, module };
1026 };
1027 defer context.dispose();
1028
1029 var target: *llvm.Target = undefined;
1030 var error_message: [*:0]const u8 = undefined;
1031 if (llvm.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {
1032 defer llvm.disposeMessage(error_message);
1033 return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message });
1034 }
1035
1036 const optimize_mode = comp.root_mod.optimize_mode;
1037
1038 const opt_level: llvm.CodeGenOptLevel = if (optimize_mode == .Debug)
1039 .None
1040 else
1041 .Aggressive;
1042
1043 const reloc_mode: llvm.RelocMode = if (comp.root_mod.pic)
1044 .PIC
1045 else if (comp.config.link_mode == .dynamic)
1046 llvm.RelocMode.DynamicNoPIC
1047 else
1048 .Static;
1049
1050 const code_model: llvm.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
1051 .default => .Default,
1052 .tiny => .Tiny,
1053 .small => .Small,
1054 .kernel => .Kernel,
1055 .medium => .Medium,
1056 .large => .Large,
1057 };
1058
1059 const float_abi: llvm.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)
1060 .Hard
1061 else
1062 .Soft;
1063
1064 var target_machine = llvm.TargetMachine.create(
1065 target,
1066 target_triple_sentinel,
1067 if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) |s| s.ptr else null,
1068 comp.root_mod.resolved_target.llvm_cpu_features.?,
1069 opt_level,
1070 reloc_mode,
1071 code_model,
1072 comp.function_sections,
1073 comp.data_sections,
1074 float_abi,
1075 if (target_util.llvmMachineAbi(&comp.root_mod.resolved_target.result)) |s| s.ptr else null,
1076 target_util.useEmulatedTls(&comp.root_mod.resolved_target.result),
1077 );
1078 errdefer target_machine.dispose();
1079
1080 if (comp.llvm_opt_bisect_limit >= 0) {
1081 context.setOptBisectLimit(comp.llvm_opt_bisect_limit);
1082 }
1083
1084 // Unfortunately, LLVM shits the bed when we ask for both binary and assembly.
1085 // So we call the entire pipeline multiple times if this is requested.
1086 // var error_message: [*:0]const u8 = undefined;
1087 var lowered_options: llvm.TargetMachine.EmitOptions = .{
1088 .is_debug = options.is_debug,
1089 .is_small = options.is_small,
1090 .time_report_out = null, // set below to make sure it's only set for a single `emitToFile`
1091 .tsan = options.sanitize_thread,
1092 .lto = switch (options.lto) {
1093 .none => .None,
1094 .thin => .ThinPreLink,
1095 .full => .FullPreLink,
1096 },
1097 .allow_fast_isel = true,
1098 // LLVM's RISC-V backend for some reason enables the machine outliner by default even
1099 // though it's clearly not ready and produces multiple miscompilations in our std tests.
1100 .allow_machine_outliner = !comp.root_mod.resolved_target.result.cpu.arch.isRISCV(),
1101 .asm_filename = null,
1102 .bin_filename = if (options.bin_path) |x| x.ptr else null,
1103 .llvm_ir_filename = if (options.post_ir_path) |x| x.ptr else null,
1104 .bitcode_filename = null,
1105
1106 // `.coverage` value is only used when `.sancov` is enabled.
1107 .sancov = options.fuzz or comp.config.san_cov_trace_pc_guard,
1108 .coverage = .{
1109 .CoverageType = .Edge,
1110 // Works in tandem with Inline8bitCounters or InlineBoolFlag.
1111 // Zig does not yet implement its own version of this but it
1112 // needs to for better fuzzing logic.
1113 .IndirectCalls = false,
1114 .TraceBB = false,
1115 .TraceCmp = options.fuzz,
1116 .TraceDiv = false,
1117 .TraceGep = false,
1118 .Use8bitCounters = false,
1119 .TracePC = false,
1120 .TracePCGuard = comp.config.san_cov_trace_pc_guard,
1121 // Zig emits its own inline 8-bit counters instrumentation.
1122 .Inline8bitCounters = false,
1123 .InlineBoolFlag = false,
1124 // Zig emits its own PC table instrumentation.
1125 .PCTable = false,
1126 .NoPrune = false,
1127 // Workaround for https://github.com/llvm/llvm-project/pull/106464
1128 .StackDepth = true,
1129 .TraceLoads = false,
1130 .TraceStores = false,
1131 .CollectControlFlow = false,
1132 },
1133 };
1134 if (options.asm_path != null and options.bin_path != null) {
1135 if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1136 defer llvm.disposeMessage(error_message);
1137 return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{
1138 emit_bin_msg, post_llvm_ir_msg, error_message,
1139 });
1140 }
1141 lowered_options.bin_filename = null;
1142 lowered_options.llvm_ir_filename = null;
1143 }
1144
1145 var time_report_c_str: [*:0]u8 = undefined;
1146 if (options.time_report != null) {
1147 lowered_options.time_report_out = &time_report_c_str;
1148 }
1149
1150 lowered_options.asm_filename = if (options.asm_path) |x| x.ptr else null;
1151 if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1152 defer llvm.disposeMessage(error_message);
1153 return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{
1154 emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message,
1155 });
1156 }
1157 if (options.time_report) |tr| {
1158 defer std.c.free(time_report_c_str);
1159 const time_report_data = std.mem.span(time_report_c_str);
1160 assert(tr.llvm_pass_timings.len == 0);
1161 tr.llvm_pass_timings = try comp.gpa.dupe(u8, time_report_data);
1162 }
1163 }
1164
1165 pub fn updateFunc(
1166 o: *Object,
1167 pt: Zcu.PerThread,
1168 func_index: InternPool.Index,
1169 air: *const Air,
1170 liveness: *const ?Air.Liveness,
1171 ) !void {
1172 const zcu = pt.zcu;
1173 const comp = zcu.comp;
1174 const ip = &zcu.intern_pool;
1175 const func = zcu.funcInfo(func_index);
1176 const nav = ip.getNav(func.owner_nav);
1177 const file_scope = zcu.navFileScopeIndex(func.owner_nav);
1178 const owner_mod = zcu.fileByIndex(file_scope).mod.?;
1179 const fn_ty = Type.fromInterned(func.ty);
1180 const fn_info = zcu.typeToFunc(fn_ty).?;
1181 const target = &owner_mod.resolved_target.result;
1182
1183 var ng: NavGen = .{
1184 .object = o,
1185 .nav_index = func.owner_nav,
1186 .pt = pt,
1187 .err_msg = null,
1188 };
1189
1190 const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
1191
1192 var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
1193 defer attributes.deinit(&o.builder);
1194
1195 const func_analysis = func.analysisUnordered(ip);
1196 if (func_analysis.is_noinline) {
1197 try attributes.addFnAttr(.@"noinline", &o.builder);
1198 } else {
1199 _ = try attributes.removeFnAttr(.@"noinline");
1200 }
1201
1202 if (func_analysis.branch_hint == .cold) {
1203 try attributes.addFnAttr(.cold, &o.builder);
1204 } else {
1205 _ = try attributes.removeFnAttr(.cold);
1206 }
1207
1208 if (owner_mod.sanitize_thread and !func_analysis.disable_instrumentation) {
1209 try attributes.addFnAttr(.sanitize_thread, &o.builder);
1210 } else {
1211 _ = try attributes.removeFnAttr(.sanitize_thread);
1212 }
1213 const is_naked = fn_info.cc == .naked;
1214 if (!func_analysis.disable_instrumentation and !is_naked) {
1215 if (owner_mod.fuzz) {
1216 try attributes.addFnAttr(.optforfuzzing, &o.builder);
1217 }
1218 _ = try attributes.removeFnAttr(.skipprofile);
1219 _ = try attributes.removeFnAttr(.nosanitize_coverage);
1220 } else {
1221 _ = try attributes.removeFnAttr(.optforfuzzing);
1222 try attributes.addFnAttr(.skipprofile, &o.builder);
1223 try attributes.addFnAttr(.nosanitize_coverage, &o.builder);
1224 }
1225
1226 const disable_intrinsics = func_analysis.disable_intrinsics or owner_mod.no_builtin;
1227 if (disable_intrinsics) {
1228 // The intent here is for compiler-rt and libc functions to not generate
1229 // infinite recursion. For example, if we are compiling the memcpy function,
1230 // and llvm detects that the body is equivalent to memcpy, it may replace the
1231 // body of memcpy with a call to memcpy, which would then cause a stack
1232 // overflow instead of performing memcpy.
1233 try attributes.addFnAttr(.{ .string = .{
1234 .kind = try o.builder.string("no-builtins"),
1235 .value = .empty,
1236 } }, &o.builder);
1237 }
1238
1239 // TODO: disable this if safety is off for the function scope
1240 const ssp_buf_size = owner_mod.stack_protector;
1241 if (ssp_buf_size != 0) {
1242 try attributes.addFnAttr(.sspstrong, &o.builder);
1243 try attributes.addFnAttr(.{ .string = .{
1244 .kind = try o.builder.string("stack-protector-buffer-size"),
1245 .value = try o.builder.fmt("{d}", .{ssp_buf_size}),
1246 } }, &o.builder);
1247 }
1248
1249 // TODO: disable this if safety is off for the function scope
1250 if (owner_mod.stack_check) {
1251 try attributes.addFnAttr(.{ .string = .{
1252 .kind = try o.builder.string("probe-stack"),
1253 .value = try o.builder.string("__zig_probe_stack"),
1254 } }, &o.builder);
1255 } else if (target.os.tag == .uefi) {
1256 try attributes.addFnAttr(.{ .string = .{
1257 .kind = try o.builder.string("no-stack-arg-probe"),
1258 .value = .empty,
1259 } }, &o.builder);
1260 }
1261
1262 if (nav.status.fully_resolved.@"linksection".toSlice(ip)) |section|
1263 function_index.setSection(try o.builder.string(section), &o.builder);
1264
1265 var deinit_wip = true;
1266 var wip = try Builder.WipFunction.init(&o.builder, .{
1267 .function = function_index,
1268 .strip = owner_mod.strip,
1269 });
1270 defer if (deinit_wip) wip.deinit();
1271 wip.cursor = .{ .block = try wip.block(0, "Entry") };
1272
1273 var llvm_arg_i: u32 = 0;
1274
1275 // This gets the LLVM values from the function and stores them in `ng.args`.
1276 const sret = firstParamSRet(fn_info, zcu, target);
1277 const ret_ptr: Builder.Value = if (sret) param: {
1278 const param = wip.arg(llvm_arg_i);
1279 llvm_arg_i += 1;
1280 break :param param;
1281 } else .none;
1282
1283 if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {
1284 .signed => try attributes.addRetAttr(.signext, &o.builder),
1285 .unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
1286 };
1287
1288 const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1289
1290 const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1291 const param = wip.arg(llvm_arg_i);
1292 llvm_arg_i += 1;
1293 break :param param;
1294 } else .none;
1295
1296 // This is the list of args we will use that correspond directly to the AIR arg
1297 // instructions. Depending on the calling convention, this list is not necessarily
1298 // a bijection with the actual LLVM parameters of the function.
1299 const gpa = o.gpa;
1300 var args: std.ArrayList(Builder.Value) = .empty;
1301 defer args.deinit(gpa);
1302
1303 {
1304 var it = iterateParamTypes(o, pt, fn_info);
1305 while (try it.next()) |lowering| {
1306 try args.ensureUnusedCapacity(gpa, 1);
1307
1308 switch (lowering) {
1309 .no_bits => continue,
1310 .byval => {
1311 assert(!it.byval_attr);
1312 const param_index = it.zig_index - 1;
1313 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1314 const param = wip.arg(llvm_arg_i);
1315
1316 if (isByRef(param_ty, zcu)) {
1317 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1318 const param_llvm_ty = param.typeOfWip(&wip);
1319 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1320 _ = try wip.store(.normal, param, arg_ptr, alignment);
1321 args.appendAssumeCapacity(arg_ptr);
1322 } else {
1323 args.appendAssumeCapacity(param);
1324
1325 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, llvm_arg_i);
1326 }
1327 llvm_arg_i += 1;
1328 },
1329 .byref => {
1330 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1331 const param_llvm_ty = try o.lowerType(pt, param_ty);
1332 const param = wip.arg(llvm_arg_i);
1333 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1334
1335 try o.addByRefParamAttrs(&attributes, llvm_arg_i, alignment, it.byval_attr, param_llvm_ty);
1336 llvm_arg_i += 1;
1337
1338 if (isByRef(param_ty, zcu)) {
1339 args.appendAssumeCapacity(param);
1340 } else {
1341 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1342 }
1343 },
1344 .byref_mut => {
1345 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1346 const param_llvm_ty = try o.lowerType(pt, param_ty);
1347 const param = wip.arg(llvm_arg_i);
1348 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1349
1350 try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder);
1351 llvm_arg_i += 1;
1352
1353 if (isByRef(param_ty, zcu)) {
1354 args.appendAssumeCapacity(param);
1355 } else {
1356 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1357 }
1358 },
1359 .abi_sized_int => {
1360 assert(!it.byval_attr);
1361 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1362 const param = wip.arg(llvm_arg_i);
1363 llvm_arg_i += 1;
1364
1365 const param_llvm_ty = try o.lowerType(pt, param_ty);
1366 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1367 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1368 _ = try wip.store(.normal, param, arg_ptr, alignment);
1369
1370 args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1371 arg_ptr
1372 else
1373 try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1374 },
1375 .slice => {
1376 assert(!it.byval_attr);
1377 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1378 const ptr_info = param_ty.ptrInfo(zcu);
1379
1380 if (math.cast(u5, it.zig_index - 1)) |i| {
1381 if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
1382 try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
1383 }
1384 }
1385 if (param_ty.zigTypeTag(zcu) != .optional and
1386 !ptr_info.flags.is_allowzero and
1387 ptr_info.flags.address_space == .generic)
1388 {
1389 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
1390 }
1391 if (ptr_info.flags.is_const) {
1392 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
1393 }
1394 const elem_align = (if (ptr_info.flags.alignment != .none)
1395 @as(InternPool.Alignment, ptr_info.flags.alignment)
1396 else
1397 Type.fromInterned(ptr_info.child).abiAlignment(zcu).max(.@"1")).toLlvm();
1398 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
1399 const ptr_param = wip.arg(llvm_arg_i);
1400 llvm_arg_i += 1;
1401 const len_param = wip.arg(llvm_arg_i);
1402 llvm_arg_i += 1;
1403
1404 const slice_llvm_ty = try o.lowerType(pt, param_ty);
1405 args.appendAssumeCapacity(
1406 try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""),
1407 );
1408 },
1409 .multiple_llvm_types => {
1410 assert(!it.byval_attr);
1411 const field_types = it.types_buffer[0..it.types_len];
1412 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1413 const param_llvm_ty = try o.lowerType(pt, param_ty);
1414 const param_alignment = param_ty.abiAlignment(zcu).toLlvm();
1415 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target);
1416 const llvm_ty = try o.builder.structType(.normal, field_types);
1417 for (0..field_types.len) |field_i| {
1418 const param = wip.arg(llvm_arg_i);
1419 llvm_arg_i += 1;
1420 const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, "");
1421 const alignment =
1422 Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
1423 _ = try wip.store(.normal, param, field_ptr, alignment);
1424 }
1425
1426 const is_by_ref = isByRef(param_ty, zcu);
1427 args.appendAssumeCapacity(if (is_by_ref)
1428 arg_ptr
1429 else
1430 try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, ""));
1431 },
1432 .float_array => {
1433 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1434 const param_llvm_ty = try o.lowerType(pt, param_ty);
1435 const param = wip.arg(llvm_arg_i);
1436 llvm_arg_i += 1;
1437
1438 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1439 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1440 _ = try wip.store(.normal, param, arg_ptr, alignment);
1441
1442 args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1443 arg_ptr
1444 else
1445 try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1446 },
1447 .i32_array, .i64_array => {
1448 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1449 const param_llvm_ty = try o.lowerType(pt, param_ty);
1450 const param = wip.arg(llvm_arg_i);
1451 llvm_arg_i += 1;
1452
1453 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1454 const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1455 _ = try wip.store(.normal, param, arg_ptr, alignment);
1456
1457 args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1458 arg_ptr
1459 else
1460 try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1461 },
1462 }
1463 }
1464 }
1465
1466 const file, const subprogram = if (!wip.strip) debug_info: {
1467 const file = try o.getDebugFile(pt, file_scope);
1468
1469 const line_number = zcu.navSrcLine(func.owner_nav) + 1;
1470 const is_internal_linkage = ip.indexToKey(nav.status.fully_resolved.val) != .@"extern";
1471 const debug_decl_type = try o.lowerDebugType(pt, fn_ty);
1472
1473 const subprogram = try o.builder.debugSubprogram(
1474 file,
1475 try o.builder.metadataString(nav.name.toSlice(ip)),
1476 try o.builder.metadataStringFromStrtabString(function_index.name(&o.builder)),
1477 line_number,
1478 line_number + func.lbrace_line,
1479 debug_decl_type,
1480 .{
1481 .di_flags = .{
1482 .StaticMember = true,
1483 .NoReturn = fn_info.return_type == .noreturn_type,
1484 },
1485 .sp_flags = .{
1486 .Optimized = owner_mod.optimize_mode != .Debug,
1487 .Definition = true,
1488 .LocalToUnit = is_internal_linkage,
1489 },
1490 },
1491 o.debug_compile_unit.unwrap().?,
1492 );
1493 function_index.setSubprogram(subprogram, &o.builder);
1494 break :debug_info .{ file, subprogram };
1495 } else .{undefined} ** 2;
1496
1497 const fuzz: ?FuncGen.Fuzz = f: {
1498 if (!owner_mod.fuzz) break :f null;
1499 if (func_analysis.disable_instrumentation) break :f null;
1500 if (is_naked) break :f null;
1501 if (comp.config.san_cov_trace_pc_guard) break :f null;
1502
1503 // The void type used here is a placeholder to be replaced with an
1504 // array of the appropriate size after the POI count is known.
1505
1506 // Due to error "members of llvm.compiler.used must be named", this global needs a name.
1507 const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
1508 const counters_variable = try o.builder.addVariable(anon_name, .void, .default);
1509 try o.used.append(gpa, counters_variable.toConst(&o.builder));
1510 counters_variable.setLinkage(.private, &o.builder);
1511 counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
1512
1513 if (target.ofmt == .macho) {
1514 counters_variable.setSection(try o.builder.string("__DATA,__sancov_cntrs"), &o.builder);
1515 } else {
1516 counters_variable.setSection(try o.builder.string("__sancov_cntrs"), &o.builder);
1517 }
1518
1519 break :f .{
1520 .counters_variable = counters_variable,
1521 .pcs = .{},
1522 };
1523 };
1524
1525 var fg: FuncGen = .{
1526 .gpa = gpa,
1527 .air = air.*,
1528 .liveness = liveness.*.?,
1529 .ng = &ng,
1530 .wip = wip,
1531 .is_naked = fn_info.cc == .naked,
1532 .fuzz = fuzz,
1533 .ret_ptr = ret_ptr,
1534 .args = args.items,
1535 .arg_index = 0,
1536 .arg_inline_index = 0,
1537 .func_inst_table = .{},
1538 .blocks = .{},
1539 .loops = .{},
1540 .switch_dispatch_info = .{},
1541 .sync_scope = if (owner_mod.single_threaded) .singlethread else .system,
1542 .file = file,
1543 .scope = subprogram,
1544 .base_line = zcu.navSrcLine(func.owner_nav),
1545 .prev_dbg_line = 0,
1546 .prev_dbg_column = 0,
1547 .err_ret_trace = err_ret_trace,
1548 .disable_intrinsics = disable_intrinsics,
1549 };
1550 defer fg.deinit();
1551 deinit_wip = false;
1552
1553 fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) {
1554 error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) {
1555 error.CodegenFail => return,
1556 error.OutOfMemory => |e| return e,
1557 },
1558 else => |e| return e,
1559 };
1560
1561 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
1562 // function as considering null pointers valid so that LLVM's optimizers don't remove these
1563 // operations on the assumption that they're undefined behavior.
1564 if (fg.allowzero_access) {
1565 try attributes.addFnAttr(.null_pointer_is_valid, &o.builder);
1566 } else {
1567 _ = try attributes.removeFnAttr(.null_pointer_is_valid);
1568 }
1569
1570 function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
1571
1572 if (fg.fuzz) |*f| {
1573 {
1574 const array_llvm_ty = try o.builder.arrayType(f.pcs.items.len, .i8);
1575 f.counters_variable.ptrConst(&o.builder).global.ptr(&o.builder).type = array_llvm_ty;
1576 const zero_init = try o.builder.zeroInitConst(array_llvm_ty);
1577 try f.counters_variable.setInitializer(zero_init, &o.builder);
1578 }
1579
1580 const array_llvm_ty = try o.builder.arrayType(f.pcs.items.len, .ptr);
1581 const init_val = try o.builder.arrayConst(array_llvm_ty, f.pcs.items);
1582 // Due to error "members of llvm.compiler.used must be named", this global needs a name.
1583 const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
1584 const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default);
1585 try o.used.append(gpa, pcs_variable.toConst(&o.builder));
1586 pcs_variable.setLinkage(.private, &o.builder);
1587 pcs_variable.setMutability(.constant, &o.builder);
1588 pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder);
1589 if (target.ofmt == .macho) {
1590 pcs_variable.setSection(try o.builder.string("__DATA,__sancov_pcs1"), &o.builder);
1591 } else {
1592 pcs_variable.setSection(try o.builder.string("__sancov_pcs1"), &o.builder);
1593 }
1594 try pcs_variable.setInitializer(init_val, &o.builder);
1595 }
1596
1597 try fg.wip.finish();
1598 }
1599
1600 pub fn updateNav(self: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {
1601 var ng: NavGen = .{
1602 .object = self,
1603 .nav_index = nav_index,
1604 .pt = pt,
1605 .err_msg = null,
1606 };
1607 ng.genDecl() catch |err| switch (err) {
1608 error.CodegenFail => switch (pt.zcu.codegenFailMsg(nav_index, ng.err_msg.?)) {
1609 error.CodegenFail => return,
1610 error.OutOfMemory => |e| return e,
1611 },
1612 else => |e| return e,
1613 };
1614 }
1615
1616 pub fn updateExports(
1617 self: *Object,
1618 pt: Zcu.PerThread,
1619 exported: Zcu.Exported,
1620 export_indices: []const Zcu.Export.Index,
1621 ) link.File.UpdateExportsError!void {
1622 const zcu = pt.zcu;
1623 const nav_index = switch (exported) {
1624 .nav => |nav| nav,
1625 .uav => |uav| return updateExportedValue(self, pt, uav, export_indices),
1626 };
1627 const ip = &zcu.intern_pool;
1628 const global_index = self.nav_map.get(nav_index).?;
1629 const comp = zcu.comp;
1630
1631 // If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use.
1632 coff_export_flags: {
1633 const lf = comp.bin_file orelse break :coff_export_flags;
1634 const lld = lf.cast(.lld) orelse break :coff_export_flags;
1635 const coff = switch (lld.ofmt) {
1636 .elf, .wasm => break :coff_export_flags,
1637 .coff => |*coff| coff,
1638 };
1639 if (!ip.isFunctionType(ip.getNav(nav_index).typeOf(ip))) break :coff_export_flags;
1640 const flags = &coff.lld_export_flags;
1641 for (export_indices) |export_index| {
1642 const name = export_index.ptr(zcu).opts.name;
1643 if (name.eqlSlice("main", ip)) flags.c_main = true;
1644 if (name.eqlSlice("WinMain", ip)) flags.winmain = true;
1645 if (name.eqlSlice("wWinMain", ip)) flags.wwinmain = true;
1646 if (name.eqlSlice("WinMainCRTStartup", ip)) flags.winmain_crt_startup = true;
1647 if (name.eqlSlice("wWinMainCRTStartup", ip)) flags.wwinmain_crt_startup = true;
1648 if (name.eqlSlice("DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;
1649 }
1650 }
1651
1652 if (export_indices.len != 0) {
1653 return updateExportedGlobal(self, zcu, global_index, export_indices);
1654 } else {
1655 const fqn = try self.builder.strtabString(ip.getNav(nav_index).fqn.toSlice(ip));
1656 try global_index.rename(fqn, &self.builder);
1657 global_index.setLinkage(.internal, &self.builder);
1658 if (comp.config.dll_export_fns)
1659 global_index.setDllStorageClass(.default, &self.builder);
1660 global_index.setUnnamedAddr(.unnamed_addr, &self.builder);
1661 }
1662 }
1663
1664 fn updateExportedValue(
1665 o: *Object,
1666 pt: Zcu.PerThread,
1667 exported_value: InternPool.Index,
1668 export_indices: []const Zcu.Export.Index,
1669 ) link.File.UpdateExportsError!void {
1670 const zcu = pt.zcu;
1671 const gpa = zcu.gpa;
1672 const ip = &zcu.intern_pool;
1673 const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip));
1674 const global_index = i: {
1675 const gop = try o.uav_map.getOrPut(gpa, exported_value);
1676 if (gop.found_existing) {
1677 const global_index = gop.value_ptr.*;
1678 try global_index.rename(main_exp_name, &o.builder);
1679 break :i global_index;
1680 }
1681 const llvm_addr_space = toLlvmAddressSpace(.generic, o.target);
1682 const variable_index = try o.builder.addVariable(
1683 main_exp_name,
1684 try o.lowerType(pt, Type.fromInterned(ip.typeOf(exported_value))),
1685 llvm_addr_space,
1686 );
1687 const global_index = variable_index.ptrConst(&o.builder).global;
1688 gop.value_ptr.* = global_index;
1689 // This line invalidates `gop`.
1690 const init_val = o.lowerValue(pt, exported_value) catch |err| switch (err) {
1691 error.OutOfMemory => return error.OutOfMemory,
1692 error.CodegenFail => return error.AnalysisFail,
1693 };
1694 try variable_index.setInitializer(init_val, &o.builder);
1695 break :i global_index;
1696 };
1697 return updateExportedGlobal(o, zcu, global_index, export_indices);
1698 }
1699
1700 fn updateExportedGlobal(
1701 o: *Object,
1702 zcu: *Zcu,
1703 global_index: Builder.Global.Index,
1704 export_indices: []const Zcu.Export.Index,
1705 ) link.File.UpdateExportsError!void {
1706 const comp = zcu.comp;
1707 const ip = &zcu.intern_pool;
1708 const first_export = export_indices[0].ptr(zcu);
1709
1710 // We will rename this global to have a name matching `first_export`.
1711 // Successive exports become aliases.
1712 // If the first export name already exists, then there is a corresponding
1713 // extern global - we replace it with this global.
1714 const first_exp_name = try o.builder.strtabString(first_export.opts.name.toSlice(ip));
1715 if (o.builder.getGlobal(first_exp_name)) |other_global| replace: {
1716 if (other_global.toConst().getBase(&o.builder) == global_index.toConst().getBase(&o.builder)) {
1717 break :replace; // this global already has the name we want
1718 }
1719 try global_index.takeName(other_global, &o.builder);
1720 try other_global.replace(global_index, &o.builder);
1721 // Problem: now we need to replace in the decl_map that
1722 // the extern decl index points to this new global. However we don't
1723 // know the decl index.
1724 // Even if we did, a future incremental update to the extern would then
1725 // treat the LLVM global as an extern rather than an export, so it would
1726 // need a way to check that.
1727 // This is a TODO that needs to be solved when making
1728 // the LLVM backend support incremental compilation.
1729 } else {
1730 try global_index.rename(first_exp_name, &o.builder);
1731 }
1732
1733 global_index.setUnnamedAddr(.default, &o.builder);
1734 if (comp.config.dll_export_fns)
1735 global_index.setDllStorageClass(.dllexport, &o.builder);
1736 global_index.setLinkage(switch (first_export.opts.linkage) {
1737 .internal => unreachable,
1738 .strong => .external,
1739 .weak => .weak_odr,
1740 .link_once => .linkonce_odr,
1741 }, &o.builder);
1742 global_index.setVisibility(switch (first_export.opts.visibility) {
1743 .default => .default,
1744 .hidden => .hidden,
1745 .protected => .protected,
1746 }, &o.builder);
1747 if (first_export.opts.section.toSlice(ip)) |section|
1748 switch (global_index.ptrConst(&o.builder).kind) {
1749 .variable => |impl_index| impl_index.setSection(
1750 try o.builder.string(section),
1751 &o.builder,
1752 ),
1753 .function => unreachable,
1754 .alias => unreachable,
1755 .replaced => unreachable,
1756 };
1757
1758 // If a Decl is exported more than one time (which is rare),
1759 // we add aliases for all but the first export.
1760 // TODO LLVM C API does not support deleting aliases.
1761 // The planned solution to this is https://github.com/ziglang/zig/issues/13265
1762 // Until then we iterate over existing aliases and make them point
1763 // to the correct decl, or otherwise add a new alias. Old aliases are leaked.
1764 for (export_indices[1..]) |export_idx| {
1765 const exp = export_idx.ptr(zcu);
1766 const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip));
1767 if (o.builder.getGlobal(exp_name)) |global| {
1768 switch (global.ptrConst(&o.builder).kind) {
1769 .alias => |alias| {
1770 alias.setAliasee(global_index.toConst(), &o.builder);
1771 continue;
1772 },
1773 .variable, .function => {
1774 // This existing global is an `extern` corresponding to this export.
1775 // Replace it with the global being exported.
1776 // This existing global must be replaced with the alias.
1777 try global.rename(.empty, &o.builder);
1778 try global.replace(global_index, &o.builder);
1779 },
1780 .replaced => unreachable,
1781 }
1782 }
1783 const alias_index = try o.builder.addAlias(
1784 .empty,
1785 global_index.typeOf(&o.builder),
1786 .default,
1787 global_index.toConst(),
1788 );
1789 try alias_index.rename(exp_name, &o.builder);
1790 }
1791 }
1792
1793 fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {
1794 const gpa = o.gpa;
1795 const gop = try o.debug_file_map.getOrPut(gpa, file_index);
1796 errdefer assert(o.debug_file_map.remove(file_index));
1797 if (gop.found_existing) return gop.value_ptr.*;
1798 const path = pt.zcu.fileByIndex(file_index).path;
1799 const abs_path = try path.toAbsolute(pt.zcu.comp.dirs, gpa);
1800 defer gpa.free(abs_path);
1801
1802 gop.value_ptr.* = try o.builder.debugFile(
1803 try o.builder.metadataString(std.fs.path.basename(abs_path)),
1804 try o.builder.metadataString(std.fs.path.dirname(abs_path) orelse ""),
1805 );
1806 return gop.value_ptr.*;
1807 }
1808
1809 pub fn lowerDebugType(
1810 o: *Object,
1811 pt: Zcu.PerThread,
1812 ty: Type,
1813 ) Allocator.Error!Builder.Metadata {
1814 assert(!o.builder.strip);
1815
1816 const gpa = o.gpa;
1817 const target = o.target;
1818 const zcu = pt.zcu;
1819 const ip = &zcu.intern_pool;
1820
1821 if (o.debug_type_map.get(ty.toIntern())) |debug_type| return debug_type;
1822
1823 switch (ty.zigTypeTag(zcu)) {
1824 .void,
1825 .noreturn,
1826 => {
1827 const debug_void_type = try o.builder.debugSignedType(
1828 try o.builder.metadataString("void"),
1829 0,
1830 );
1831 try o.debug_type_map.put(gpa, ty.toIntern(), debug_void_type);
1832 return debug_void_type;
1833 },
1834 .int => {
1835 const info = ty.intInfo(zcu);
1836 assert(info.bits != 0);
1837 const name = try o.allocTypeName(pt, ty);
1838 defer gpa.free(name);
1839 const builder_name = try o.builder.metadataString(name);
1840 const debug_bits = ty.abiSize(zcu) * 8; // lldb cannot handle non-byte sized types
1841 const debug_int_type = switch (info.signedness) {
1842 .signed => try o.builder.debugSignedType(builder_name, debug_bits),
1843 .unsigned => try o.builder.debugUnsignedType(builder_name, debug_bits),
1844 };
1845 try o.debug_type_map.put(gpa, ty.toIntern(), debug_int_type);
1846 return debug_int_type;
1847 },
1848 .@"enum" => {
1849 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {
1850 const debug_enum_type = try o.makeEmptyNamespaceDebugType(pt, ty);
1851 try o.debug_type_map.put(gpa, ty.toIntern(), debug_enum_type);
1852 return debug_enum_type;
1853 }
1854
1855 const enum_type = ip.loadEnumType(ty.toIntern());
1856 const enumerators = try gpa.alloc(Builder.Metadata, enum_type.names.len);
1857 defer gpa.free(enumerators);
1858
1859 const int_ty = Type.fromInterned(enum_type.tag_ty);
1860 const int_info = ty.intInfo(zcu);
1861 assert(int_info.bits != 0);
1862
1863 for (enum_type.names.get(ip), 0..) |field_name_ip, i| {
1864 var bigint_space: Value.BigIntSpace = undefined;
1865 const bigint = if (enum_type.values.len != 0)
1866 Value.fromInterned(enum_type.values.get(ip)[i]).toBigInt(&bigint_space, zcu)
1867 else
1868 std.math.big.int.Mutable.init(&bigint_space.limbs, i).toConst();
1869
1870 enumerators[i] = try o.builder.debugEnumerator(
1871 try o.builder.metadataString(field_name_ip.toSlice(ip)),
1872 int_info.signedness == .unsigned,
1873 int_info.bits,
1874 bigint,
1875 );
1876 }
1877
1878 const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
1879 const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
1880 try o.namespaceToDebugScope(pt, parent_namespace)
1881 else
1882 file;
1883
1884 const name = try o.allocTypeName(pt, ty);
1885 defer gpa.free(name);
1886
1887 const debug_enum_type = try o.builder.debugEnumerationType(
1888 try o.builder.metadataString(name),
1889 file,
1890 scope,
1891 ty.typeDeclSrcLine(zcu).? + 1, // Line
1892 try o.lowerDebugType(pt, int_ty),
1893 ty.abiSize(zcu) * 8,
1894 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
1895 try o.builder.metadataTuple(enumerators),
1896 );
1897
1898 try o.debug_type_map.put(gpa, ty.toIntern(), debug_enum_type);
1899 try o.debug_enums.append(gpa, debug_enum_type);
1900 return debug_enum_type;
1901 },
1902 .float => {
1903 const bits = ty.floatBits(target);
1904 const name = try o.allocTypeName(pt, ty);
1905 defer gpa.free(name);
1906 const debug_float_type = try o.builder.debugFloatType(
1907 try o.builder.metadataString(name),
1908 bits,
1909 );
1910 try o.debug_type_map.put(gpa, ty.toIntern(), debug_float_type);
1911 return debug_float_type;
1912 },
1913 .bool => {
1914 const debug_bool_type = try o.builder.debugBoolType(
1915 try o.builder.metadataString("bool"),
1916 8, // lldb cannot handle non-byte sized types
1917 );
1918 try o.debug_type_map.put(gpa, ty.toIntern(), debug_bool_type);
1919 return debug_bool_type;
1920 },
1921 .pointer => {
1922 // Normalize everything that the debug info does not represent.
1923 const ptr_info = ty.ptrInfo(zcu);
1924
1925 if (ptr_info.sentinel != .none or
1926 ptr_info.flags.address_space != .generic or
1927 ptr_info.packed_offset.bit_offset != 0 or
1928 ptr_info.packed_offset.host_size != 0 or
1929 ptr_info.flags.vector_index != .none or
1930 ptr_info.flags.is_allowzero or
1931 ptr_info.flags.is_const or
1932 ptr_info.flags.is_volatile or
1933 ptr_info.flags.size == .many or ptr_info.flags.size == .c or
1934 !Type.fromInterned(ptr_info.child).hasRuntimeBitsIgnoreComptime(zcu))
1935 {
1936 const bland_ptr_ty = try pt.ptrType(.{
1937 .child = if (!Type.fromInterned(ptr_info.child).hasRuntimeBitsIgnoreComptime(zcu))
1938 .anyopaque_type
1939 else
1940 ptr_info.child,
1941 .flags = .{
1942 .alignment = ptr_info.flags.alignment,
1943 .size = switch (ptr_info.flags.size) {
1944 .many, .c, .one => .one,
1945 .slice => .slice,
1946 },
1947 },
1948 });
1949 const debug_ptr_type = try o.lowerDebugType(pt, bland_ptr_ty);
1950 try o.debug_type_map.put(gpa, ty.toIntern(), debug_ptr_type);
1951 return debug_ptr_type;
1952 }
1953
1954 const debug_fwd_ref = try o.builder.debugForwardReference();
1955
1956 // Set as forward reference while the type is lowered in case it references itself
1957 try o.debug_type_map.put(gpa, ty.toIntern(), debug_fwd_ref);
1958
1959 if (ty.isSlice(zcu)) {
1960 const ptr_ty = ty.slicePtrFieldType(zcu);
1961 const len_ty = Type.usize;
1962
1963 const name = try o.allocTypeName(pt, ty);
1964 defer gpa.free(name);
1965 const line = 0;
1966
1967 const ptr_size = ptr_ty.abiSize(zcu);
1968 const ptr_align = ptr_ty.abiAlignment(zcu);
1969 const len_size = len_ty.abiSize(zcu);
1970 const len_align = len_ty.abiAlignment(zcu);
1971
1972 const len_offset = len_align.forward(ptr_size);
1973
1974 const debug_ptr_type = try o.builder.debugMemberType(
1975 try o.builder.metadataString("ptr"),
1976 null, // File
1977 debug_fwd_ref,
1978 0, // Line
1979 try o.lowerDebugType(pt, ptr_ty),
1980 ptr_size * 8,
1981 (ptr_align.toByteUnits() orelse 0) * 8,
1982 0, // Offset
1983 );
1984
1985 const debug_len_type = try o.builder.debugMemberType(
1986 try o.builder.metadataString("len"),
1987 null, // File
1988 debug_fwd_ref,
1989 0, // Line
1990 try o.lowerDebugType(pt, len_ty),
1991 len_size * 8,
1992 (len_align.toByteUnits() orelse 0) * 8,
1993 len_offset * 8,
1994 );
1995
1996 const debug_slice_type = try o.builder.debugStructType(
1997 try o.builder.metadataString(name),
1998 null, // File
1999 o.debug_compile_unit.unwrap().?, // Scope
2000 line,
2001 null, // Underlying type
2002 ty.abiSize(zcu) * 8,
2003 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2004 try o.builder.metadataTuple(&.{
2005 debug_ptr_type,
2006 debug_len_type,
2007 }),
2008 );
2009
2010 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_slice_type);
2011
2012 // Set to real type now that it has been lowered fully
2013 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2014 map_ptr.* = debug_slice_type;
2015
2016 return debug_slice_type;
2017 }
2018
2019 const debug_elem_ty = try o.lowerDebugType(pt, Type.fromInterned(ptr_info.child));
2020
2021 const name = try o.allocTypeName(pt, ty);
2022 defer gpa.free(name);
2023
2024 const debug_ptr_type = try o.builder.debugPointerType(
2025 try o.builder.metadataString(name),
2026 null, // File
2027 null, // Scope
2028 0, // Line
2029 debug_elem_ty,
2030 target.ptrBitWidth(),
2031 (ty.ptrAlignment(zcu).toByteUnits() orelse 0) * 8,
2032 0, // Offset
2033 );
2034
2035 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_ptr_type);
2036
2037 // Set to real type now that it has been lowered fully
2038 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2039 map_ptr.* = debug_ptr_type;
2040
2041 return debug_ptr_type;
2042 },
2043 .@"opaque" => {
2044 if (ty.toIntern() == .anyopaque_type) {
2045 const debug_opaque_type = try o.builder.debugSignedType(
2046 try o.builder.metadataString("anyopaque"),
2047 0,
2048 );
2049 try o.debug_type_map.put(gpa, ty.toIntern(), debug_opaque_type);
2050 return debug_opaque_type;
2051 }
2052
2053 const name = try o.allocTypeName(pt, ty);
2054 defer gpa.free(name);
2055
2056 const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2057 const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
2058 try o.namespaceToDebugScope(pt, parent_namespace)
2059 else
2060 file;
2061
2062 const debug_opaque_type = try o.builder.debugStructType(
2063 try o.builder.metadataString(name),
2064 file,
2065 scope,
2066 ty.typeDeclSrcLine(zcu).? + 1, // Line
2067 null, // Underlying type
2068 0, // Size
2069 0, // Align
2070 null, // Fields
2071 );
2072 try o.debug_type_map.put(gpa, ty.toIntern(), debug_opaque_type);
2073 return debug_opaque_type;
2074 },
2075 .array => {
2076 const debug_array_type = try o.builder.debugArrayType(
2077 null, // Name
2078 null, // File
2079 null, // Scope
2080 0, // Line
2081 try o.lowerDebugType(pt, ty.childType(zcu)),
2082 ty.abiSize(zcu) * 8,
2083 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2084 try o.builder.metadataTuple(&.{
2085 try o.builder.debugSubrange(
2086 try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
2087 try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),
2088 ),
2089 }),
2090 );
2091 try o.debug_type_map.put(gpa, ty.toIntern(), debug_array_type);
2092 return debug_array_type;
2093 },
2094 .vector => {
2095 const elem_ty = ty.elemType2(zcu);
2096 // Vector elements cannot be padded since that would make
2097 // @bitSizOf(elem) * len > @bitSizOf(vec).
2098 // Neither gdb nor lldb seem to be able to display non-byte sized
2099 // vectors properly.
2100 const debug_elem_type = switch (elem_ty.zigTypeTag(zcu)) {
2101 .int => blk: {
2102 const info = elem_ty.intInfo(zcu);
2103 assert(info.bits != 0);
2104 const name = try o.allocTypeName(pt, ty);
2105 defer gpa.free(name);
2106 const builder_name = try o.builder.metadataString(name);
2107 break :blk switch (info.signedness) {
2108 .signed => try o.builder.debugSignedType(builder_name, info.bits),
2109 .unsigned => try o.builder.debugUnsignedType(builder_name, info.bits),
2110 };
2111 },
2112 .bool => try o.builder.debugBoolType(
2113 try o.builder.metadataString("bool"),
2114 1,
2115 ),
2116 else => try o.lowerDebugType(pt, ty.childType(zcu)),
2117 };
2118
2119 const debug_vector_type = try o.builder.debugVectorType(
2120 null, // Name
2121 null, // File
2122 null, // Scope
2123 0, // Line
2124 debug_elem_type,
2125 ty.abiSize(zcu) * 8,
2126 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2127 try o.builder.metadataTuple(&.{
2128 try o.builder.debugSubrange(
2129 try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
2130 try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),
2131 ),
2132 }),
2133 );
2134
2135 try o.debug_type_map.put(gpa, ty.toIntern(), debug_vector_type);
2136 return debug_vector_type;
2137 },
2138 .optional => {
2139 const name = try o.allocTypeName(pt, ty);
2140 defer gpa.free(name);
2141 const child_ty = ty.optionalChild(zcu);
2142 if (!child_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2143 const debug_bool_type = try o.builder.debugBoolType(
2144 try o.builder.metadataString(name),
2145 8,
2146 );
2147 try o.debug_type_map.put(gpa, ty.toIntern(), debug_bool_type);
2148 return debug_bool_type;
2149 }
2150
2151 const debug_fwd_ref = try o.builder.debugForwardReference();
2152
2153 // Set as forward reference while the type is lowered in case it references itself
2154 try o.debug_type_map.put(gpa, ty.toIntern(), debug_fwd_ref);
2155
2156 if (ty.optionalReprIsPayload(zcu)) {
2157 const debug_optional_type = try o.lowerDebugType(pt, child_ty);
2158
2159 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_optional_type);
2160
2161 // Set to real type now that it has been lowered fully
2162 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2163 map_ptr.* = debug_optional_type;
2164
2165 return debug_optional_type;
2166 }
2167
2168 const non_null_ty = Type.u8;
2169 const payload_size = child_ty.abiSize(zcu);
2170 const payload_align = child_ty.abiAlignment(zcu);
2171 const non_null_size = non_null_ty.abiSize(zcu);
2172 const non_null_align = non_null_ty.abiAlignment(zcu);
2173 const non_null_offset = non_null_align.forward(payload_size);
2174
2175 const debug_data_type = try o.builder.debugMemberType(
2176 try o.builder.metadataString("data"),
2177 null, // File
2178 debug_fwd_ref,
2179 0, // Line
2180 try o.lowerDebugType(pt, child_ty),
2181 payload_size * 8,
2182 (payload_align.toByteUnits() orelse 0) * 8,
2183 0, // Offset
2184 );
2185
2186 const debug_some_type = try o.builder.debugMemberType(
2187 try o.builder.metadataString("some"),
2188 null,
2189 debug_fwd_ref,
2190 0,
2191 try o.lowerDebugType(pt, non_null_ty),
2192 non_null_size * 8,
2193 (non_null_align.toByteUnits() orelse 0) * 8,
2194 non_null_offset * 8,
2195 );
2196
2197 const debug_optional_type = try o.builder.debugStructType(
2198 try o.builder.metadataString(name),
2199 null, // File
2200 o.debug_compile_unit.unwrap().?, // Scope
2201 0, // Line
2202 null, // Underlying type
2203 ty.abiSize(zcu) * 8,
2204 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2205 try o.builder.metadataTuple(&.{
2206 debug_data_type,
2207 debug_some_type,
2208 }),
2209 );
2210
2211 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_optional_type);
2212
2213 // Set to real type now that it has been lowered fully
2214 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2215 map_ptr.* = debug_optional_type;
2216
2217 return debug_optional_type;
2218 },
2219 .error_union => {
2220 const payload_ty = ty.errorUnionPayload(zcu);
2221 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2222 // TODO: Maybe remove?
2223 const debug_error_union_type = try o.lowerDebugType(pt, Type.anyerror);
2224 try o.debug_type_map.put(gpa, ty.toIntern(), debug_error_union_type);
2225 return debug_error_union_type;
2226 }
2227
2228 const name = try o.allocTypeName(pt, ty);
2229 defer gpa.free(name);
2230
2231 const error_size = Type.anyerror.abiSize(zcu);
2232 const error_align = Type.anyerror.abiAlignment(zcu);
2233 const payload_size = payload_ty.abiSize(zcu);
2234 const payload_align = payload_ty.abiAlignment(zcu);
2235
2236 var error_index: u32 = undefined;
2237 var payload_index: u32 = undefined;
2238 var error_offset: u64 = undefined;
2239 var payload_offset: u64 = undefined;
2240 if (error_align.compare(.gt, payload_align)) {
2241 error_index = 0;
2242 payload_index = 1;
2243 error_offset = 0;
2244 payload_offset = payload_align.forward(error_size);
2245 } else {
2246 payload_index = 0;
2247 error_index = 1;
2248 payload_offset = 0;
2249 error_offset = error_align.forward(payload_size);
2250 }
2251
2252 const debug_fwd_ref = try o.builder.debugForwardReference();
2253
2254 var fields: [2]Builder.Metadata = undefined;
2255 fields[error_index] = try o.builder.debugMemberType(
2256 try o.builder.metadataString("tag"),
2257 null, // File
2258 debug_fwd_ref,
2259 0, // Line
2260 try o.lowerDebugType(pt, Type.anyerror),
2261 error_size * 8,
2262 (error_align.toByteUnits() orelse 0) * 8,
2263 error_offset * 8,
2264 );
2265 fields[payload_index] = try o.builder.debugMemberType(
2266 try o.builder.metadataString("value"),
2267 null, // File
2268 debug_fwd_ref,
2269 0, // Line
2270 try o.lowerDebugType(pt, payload_ty),
2271 payload_size * 8,
2272 (payload_align.toByteUnits() orelse 0) * 8,
2273 payload_offset * 8,
2274 );
2275
2276 const debug_error_union_type = try o.builder.debugStructType(
2277 try o.builder.metadataString(name),
2278 null, // File
2279 o.debug_compile_unit.unwrap().?, // Sope
2280 0, // Line
2281 null, // Underlying type
2282 ty.abiSize(zcu) * 8,
2283 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2284 try o.builder.metadataTuple(&fields),
2285 );
2286
2287 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_error_union_type);
2288
2289 try o.debug_type_map.put(gpa, ty.toIntern(), debug_error_union_type);
2290 return debug_error_union_type;
2291 },
2292 .error_set => {
2293 const debug_error_set = try o.builder.debugUnsignedType(
2294 try o.builder.metadataString("anyerror"),
2295 16,
2296 );
2297 try o.debug_type_map.put(gpa, ty.toIntern(), debug_error_set);
2298 return debug_error_set;
2299 },
2300 .@"struct" => {
2301 const name = try o.allocTypeName(pt, ty);
2302 defer gpa.free(name);
2303
2304 if (zcu.typeToPackedStruct(ty)) |struct_type| {
2305 const backing_int_ty = struct_type.backingIntTypeUnordered(ip);
2306 if (backing_int_ty != .none) {
2307 const info = Type.fromInterned(backing_int_ty).intInfo(zcu);
2308 const builder_name = try o.builder.metadataString(name);
2309 const debug_int_type = switch (info.signedness) {
2310 .signed => try o.builder.debugSignedType(builder_name, ty.abiSize(zcu) * 8),
2311 .unsigned => try o.builder.debugUnsignedType(builder_name, ty.abiSize(zcu) * 8),
2312 };
2313 try o.debug_type_map.put(gpa, ty.toIntern(), debug_int_type);
2314 return debug_int_type;
2315 }
2316 }
2317
2318 switch (ip.indexToKey(ty.toIntern())) {
2319 .tuple_type => |tuple| {
2320 var fields: std.ArrayList(Builder.Metadata) = .empty;
2321 defer fields.deinit(gpa);
2322
2323 try fields.ensureUnusedCapacity(gpa, tuple.types.len);
2324
2325 comptime assert(struct_layout_version == 2);
2326 var offset: u64 = 0;
2327
2328 const debug_fwd_ref = try o.builder.debugForwardReference();
2329
2330 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| {
2331 if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
2332
2333 const field_size = Type.fromInterned(field_ty).abiSize(zcu);
2334 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
2335 const field_offset = field_align.forward(offset);
2336 offset = field_offset + field_size;
2337
2338 var name_buf: [32]u8 = undefined;
2339 const field_name = std.fmt.bufPrint(&name_buf, "{d}", .{i}) catch unreachable;
2340
2341 fields.appendAssumeCapacity(try o.builder.debugMemberType(
2342 try o.builder.metadataString(field_name),
2343 null, // File
2344 debug_fwd_ref,
2345 0,
2346 try o.lowerDebugType(pt, Type.fromInterned(field_ty)),
2347 field_size * 8,
2348 (field_align.toByteUnits() orelse 0) * 8,
2349 field_offset * 8,
2350 ));
2351 }
2352
2353 const debug_struct_type = try o.builder.debugStructType(
2354 try o.builder.metadataString(name),
2355 null, // File
2356 o.debug_compile_unit.unwrap().?, // Scope
2357 0, // Line
2358 null, // Underlying type
2359 ty.abiSize(zcu) * 8,
2360 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2361 try o.builder.metadataTuple(fields.items),
2362 );
2363
2364 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_struct_type);
2365
2366 try o.debug_type_map.put(gpa, ty.toIntern(), debug_struct_type);
2367 return debug_struct_type;
2368 },
2369 .struct_type => {
2370 if (!ip.loadStructType(ty.toIntern()).haveFieldTypes(ip)) {
2371 // This can happen if a struct type makes it all the way to
2372 // flush() without ever being instantiated or referenced (even
2373 // via pointer). The only reason we are hearing about it now is
2374 // that it is being used as a namespace to put other debug types
2375 // into. Therefore we can satisfy this by making an empty namespace,
2376 // rather than changing the frontend to unnecessarily resolve the
2377 // struct field types.
2378 const debug_struct_type = try o.makeEmptyNamespaceDebugType(pt, ty);
2379 try o.debug_type_map.put(gpa, ty.toIntern(), debug_struct_type);
2380 return debug_struct_type;
2381 }
2382 },
2383 else => {},
2384 }
2385
2386 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {
2387 const debug_struct_type = try o.makeEmptyNamespaceDebugType(pt, ty);
2388 try o.debug_type_map.put(gpa, ty.toIntern(), debug_struct_type);
2389 return debug_struct_type;
2390 }
2391
2392 const struct_type = zcu.typeToStruct(ty).?;
2393
2394 var fields: std.ArrayList(Builder.Metadata) = .empty;
2395 defer fields.deinit(gpa);
2396
2397 try fields.ensureUnusedCapacity(gpa, struct_type.field_types.len);
2398
2399 const debug_fwd_ref = try o.builder.debugForwardReference();
2400
2401 // Set as forward reference while the type is lowered in case it references itself
2402 try o.debug_type_map.put(gpa, ty.toIntern(), debug_fwd_ref);
2403
2404 comptime assert(struct_layout_version == 2);
2405 var it = struct_type.iterateRuntimeOrder(ip);
2406 while (it.next()) |field_index| {
2407 const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[field_index]);
2408 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
2409 const field_size = field_ty.abiSize(zcu);
2410 const field_align = ty.fieldAlignment(field_index, zcu);
2411 const field_offset = ty.structFieldOffset(field_index, zcu);
2412 const field_name = struct_type.fieldName(ip, field_index);
2413 fields.appendAssumeCapacity(try o.builder.debugMemberType(
2414 try o.builder.metadataString(field_name.toSlice(ip)),
2415 null, // File
2416 debug_fwd_ref,
2417 0, // Line
2418 try o.lowerDebugType(pt, field_ty),
2419 field_size * 8,
2420 (field_align.toByteUnits() orelse 0) * 8,
2421 field_offset * 8,
2422 ));
2423 }
2424
2425 const debug_struct_type = try o.builder.debugStructType(
2426 try o.builder.metadataString(name),
2427 null, // File
2428 o.debug_compile_unit.unwrap().?, // Scope
2429 0, // Line
2430 null, // Underlying type
2431 ty.abiSize(zcu) * 8,
2432 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2433 try o.builder.metadataTuple(fields.items),
2434 );
2435
2436 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_struct_type);
2437
2438 // Set to real type now that it has been lowered fully
2439 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2440 map_ptr.* = debug_struct_type;
2441
2442 return debug_struct_type;
2443 },
2444 .@"union" => {
2445 const name = try o.allocTypeName(pt, ty);
2446 defer gpa.free(name);
2447
2448 const union_type = ip.loadUnionType(ty.toIntern());
2449 if (!union_type.haveFieldTypes(ip) or
2450 !ty.hasRuntimeBitsIgnoreComptime(zcu) or
2451 !union_type.haveLayout(ip))
2452 {
2453 const debug_union_type = try o.makeEmptyNamespaceDebugType(pt, ty);
2454 try o.debug_type_map.put(gpa, ty.toIntern(), debug_union_type);
2455 return debug_union_type;
2456 }
2457
2458 const layout = Type.getUnionLayout(union_type, zcu);
2459
2460 const debug_fwd_ref = try o.builder.debugForwardReference();
2461
2462 // Set as forward reference while the type is lowered in case it references itself
2463 try o.debug_type_map.put(gpa, ty.toIntern(), debug_fwd_ref);
2464
2465 if (layout.payload_size == 0) {
2466 const debug_union_type = try o.builder.debugStructType(
2467 try o.builder.metadataString(name),
2468 null, // File
2469 o.debug_compile_unit.unwrap().?, // Scope
2470 0, // Line
2471 null, // Underlying type
2472 ty.abiSize(zcu) * 8,
2473 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2474 try o.builder.metadataTuple(
2475 &.{try o.lowerDebugType(pt, Type.fromInterned(union_type.enum_tag_ty))},
2476 ),
2477 );
2478
2479 // Set to real type now that it has been lowered fully
2480 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2481 map_ptr.* = debug_union_type;
2482
2483 return debug_union_type;
2484 }
2485
2486 var fields: std.ArrayList(Builder.Metadata) = .empty;
2487 defer fields.deinit(gpa);
2488
2489 try fields.ensureUnusedCapacity(gpa, union_type.loadTagType(ip).names.len);
2490
2491 const debug_union_fwd_ref = if (layout.tag_size == 0)
2492 debug_fwd_ref
2493 else
2494 try o.builder.debugForwardReference();
2495
2496 const tag_type = union_type.loadTagType(ip);
2497
2498 for (0..tag_type.names.len) |field_index| {
2499 const field_ty = union_type.field_types.get(ip)[field_index];
2500 if (!Type.fromInterned(field_ty).hasRuntimeBitsIgnoreComptime(zcu)) continue;
2501
2502 const field_size = Type.fromInterned(field_ty).abiSize(zcu);
2503 const field_align: InternPool.Alignment = switch (union_type.flagsUnordered(ip).layout) {
2504 .@"packed" => .none,
2505 .auto, .@"extern" => ty.fieldAlignment(field_index, zcu),
2506 };
2507
2508 const field_name = tag_type.names.get(ip)[field_index];
2509 fields.appendAssumeCapacity(try o.builder.debugMemberType(
2510 try o.builder.metadataString(field_name.toSlice(ip)),
2511 null, // File
2512 debug_union_fwd_ref,
2513 0, // Line
2514 try o.lowerDebugType(pt, Type.fromInterned(field_ty)),
2515 field_size * 8,
2516 (field_align.toByteUnits() orelse 0) * 8,
2517 0, // Offset
2518 ));
2519 }
2520
2521 var union_name_buf: ?[:0]const u8 = null;
2522 defer if (union_name_buf) |buf| gpa.free(buf);
2523 const union_name = if (layout.tag_size == 0) name else name: {
2524 union_name_buf = try std.fmt.allocPrintSentinel(gpa, "{s}:Payload", .{name}, 0);
2525 break :name union_name_buf.?;
2526 };
2527
2528 const debug_union_type = try o.builder.debugUnionType(
2529 try o.builder.metadataString(union_name),
2530 null, // File
2531 o.debug_compile_unit.unwrap().?, // Scope
2532 0, // Line
2533 null, // Underlying type
2534 layout.payload_size * 8,
2535 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2536 try o.builder.metadataTuple(fields.items),
2537 );
2538
2539 o.builder.resolveDebugForwardReference(debug_union_fwd_ref, debug_union_type);
2540
2541 if (layout.tag_size == 0) {
2542 // Set to real type now that it has been lowered fully
2543 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2544 map_ptr.* = debug_union_type;
2545
2546 return debug_union_type;
2547 }
2548
2549 var tag_offset: u64 = undefined;
2550 var payload_offset: u64 = undefined;
2551 if (layout.tag_align.compare(.gte, layout.payload_align)) {
2552 tag_offset = 0;
2553 payload_offset = layout.payload_align.forward(layout.tag_size);
2554 } else {
2555 payload_offset = 0;
2556 tag_offset = layout.tag_align.forward(layout.payload_size);
2557 }
2558
2559 const debug_tag_type = try o.builder.debugMemberType(
2560 try o.builder.metadataString("tag"),
2561 null, // File
2562 debug_fwd_ref,
2563 0, // Line
2564 try o.lowerDebugType(pt, Type.fromInterned(union_type.enum_tag_ty)),
2565 layout.tag_size * 8,
2566 (layout.tag_align.toByteUnits() orelse 0) * 8,
2567 tag_offset * 8,
2568 );
2569
2570 const debug_payload_type = try o.builder.debugMemberType(
2571 try o.builder.metadataString("payload"),
2572 null, // File
2573 debug_fwd_ref,
2574 0, // Line
2575 debug_union_type,
2576 layout.payload_size * 8,
2577 (layout.payload_align.toByteUnits() orelse 0) * 8,
2578 payload_offset * 8,
2579 );
2580
2581 const full_fields: [2]Builder.Metadata =
2582 if (layout.tag_align.compare(.gte, layout.payload_align))
2583 .{ debug_tag_type, debug_payload_type }
2584 else
2585 .{ debug_payload_type, debug_tag_type };
2586
2587 const debug_tagged_union_type = try o.builder.debugStructType(
2588 try o.builder.metadataString(name),
2589 null, // File
2590 o.debug_compile_unit.unwrap().?, // Scope
2591 0, // Line
2592 null, // Underlying type
2593 ty.abiSize(zcu) * 8,
2594 (ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2595 try o.builder.metadataTuple(&full_fields),
2596 );
2597
2598 o.builder.resolveDebugForwardReference(debug_fwd_ref, debug_tagged_union_type);
2599
2600 // Set to real type now that it has been lowered fully
2601 const map_ptr = o.debug_type_map.getPtr(ty.toIntern()) orelse unreachable;
2602 map_ptr.* = debug_tagged_union_type;
2603
2604 return debug_tagged_union_type;
2605 },
2606 .@"fn" => {
2607 const fn_info = zcu.typeToFunc(ty).?;
2608
2609 var debug_param_types = std.array_list.Managed(Builder.Metadata).init(gpa);
2610 defer debug_param_types.deinit();
2611
2612 try debug_param_types.ensureUnusedCapacity(3 + fn_info.param_types.len);
2613
2614 // Return type goes first.
2615 if (Type.fromInterned(fn_info.return_type).hasRuntimeBitsIgnoreComptime(zcu)) {
2616 const sret = firstParamSRet(fn_info, zcu, target);
2617 const ret_ty = if (sret) Type.void else Type.fromInterned(fn_info.return_type);
2618 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, ret_ty));
2619
2620 if (sret) {
2621 const ptr_ty = try pt.singleMutPtrType(Type.fromInterned(fn_info.return_type));
2622 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, ptr_ty));
2623 }
2624 } else {
2625 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, Type.void));
2626 }
2627
2628 if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
2629 // Stack trace pointer.
2630 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, .fromInterned(.ptr_usize_type)));
2631 }
2632
2633 for (0..fn_info.param_types.len) |i| {
2634 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[i]);
2635 if (!param_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
2636
2637 if (isByRef(param_ty, zcu)) {
2638 const ptr_ty = try pt.singleMutPtrType(param_ty);
2639 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, ptr_ty));
2640 } else {
2641 debug_param_types.appendAssumeCapacity(try o.lowerDebugType(pt, param_ty));
2642 }
2643 }
2644
2645 const debug_function_type = try o.builder.debugSubroutineType(
2646 try o.builder.metadataTuple(debug_param_types.items),
2647 );
2648
2649 try o.debug_type_map.put(gpa, ty.toIntern(), debug_function_type);
2650 return debug_function_type;
2651 },
2652 .comptime_int => unreachable,
2653 .comptime_float => unreachable,
2654 .type => unreachable,
2655 .undefined => unreachable,
2656 .null => unreachable,
2657 .enum_literal => unreachable,
2658
2659 .frame => @panic("TODO implement lowerDebugType for Frame types"),
2660 .@"anyframe" => @panic("TODO implement lowerDebugType for AnyFrame types"),
2661 }
2662 }
2663
2664 fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata {
2665 const zcu = pt.zcu;
2666 const namespace = zcu.namespacePtr(namespace_index);
2667 if (namespace.parent == .none) return try o.getDebugFile(pt, namespace.file_scope);
2668
2669 const gop = try o.debug_unresolved_namespace_scopes.getOrPut(o.gpa, namespace_index);
2670
2671 if (!gop.found_existing) gop.value_ptr.* = try o.builder.debugForwardReference();
2672
2673 return gop.value_ptr.*;
2674 }
2675
2676 fn makeEmptyNamespaceDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) !Builder.Metadata {
2677 const zcu = pt.zcu;
2678 const ip = &zcu.intern_pool;
2679 const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2680 const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
2681 try o.namespaceToDebugScope(pt, parent_namespace)
2682 else
2683 file;
2684 return o.builder.debugStructType(
2685 try o.builder.metadataString(ty.containerTypeName(ip).toSlice(ip)), // TODO use fully qualified name
2686 file,
2687 scope,
2688 ty.typeDeclSrcLine(zcu).? + 1,
2689 null,
2690 0,
2691 0,
2692 null,
2693 );
2694 }
2695
2696 fn allocTypeName(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error![:0]const u8 {
2697 var aw: std.Io.Writer.Allocating = .init(o.gpa);
2698 defer aw.deinit();
2699 ty.print(&aw.writer, pt, null) catch |err| switch (err) {
2700 error.WriteFailed => return error.OutOfMemory,
2701 };
2702 return aw.toOwnedSliceSentinel(0);
2703 }
2704
2705 /// If the llvm function does not exist, create it.
2706 /// Note that this can be called before the function's semantic analysis has
2707 /// completed, so if any attributes rely on that, they must be done in updateFunc, not here.
2708 fn resolveLlvmFunction(
2709 o: *Object,
2710 pt: Zcu.PerThread,
2711 nav_index: InternPool.Nav.Index,
2712 ) Allocator.Error!Builder.Function.Index {
2713 const zcu = pt.zcu;
2714 const ip = &zcu.intern_pool;
2715 const gpa = o.gpa;
2716 const nav = ip.getNav(nav_index);
2717 const owner_mod = zcu.navFileScope(nav_index).mod.?;
2718 const ty: Type = .fromInterned(nav.typeOf(ip));
2719 const gop = try o.nav_map.getOrPut(gpa, nav_index);
2720 if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.function;
2721
2722 const fn_info = zcu.typeToFunc(ty).?;
2723 const target = &owner_mod.resolved_target.result;
2724 const sret = firstParamSRet(fn_info, zcu, target);
2725
2726 const is_extern, const lib_name = if (nav.getExtern(ip)) |@"extern"|
2727 .{ true, @"extern".lib_name }
2728 else
2729 .{ false, .none };
2730 const function_index = try o.builder.addFunction(
2731 try o.lowerType(pt, ty),
2732 try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)),
2733 toLlvmAddressSpace(nav.getAddrspace(), target),
2734 );
2735 gop.value_ptr.* = function_index.ptrConst(&o.builder).global;
2736
2737 var attributes: Builder.FunctionAttributes.Wip = .{};
2738 defer attributes.deinit(&o.builder);
2739
2740 if (!is_extern) {
2741 function_index.setLinkage(.internal, &o.builder);
2742 function_index.setUnnamedAddr(.unnamed_addr, &o.builder);
2743 } else {
2744 if (target.cpu.arch.isWasm()) {
2745 try attributes.addFnAttr(.{ .string = .{
2746 .kind = try o.builder.string("wasm-import-name"),
2747 .value = try o.builder.string(nav.name.toSlice(ip)),
2748 } }, &o.builder);
2749 if (lib_name.toSlice(ip)) |lib_name_slice| {
2750 if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{
2751 .kind = try o.builder.string("wasm-import-module"),
2752 .value = try o.builder.string(lib_name_slice),
2753 } }, &o.builder);
2754 }
2755 }
2756 }
2757
2758 var llvm_arg_i: u32 = 0;
2759 if (sret) {
2760 // Sret pointers must not be address 0
2761 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
2762 try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
2763
2764 const raw_llvm_ret_ty = try o.lowerType(pt, Type.fromInterned(fn_info.return_type));
2765 try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2766
2767 llvm_arg_i += 1;
2768 }
2769
2770 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
2771
2772 if (err_return_tracing) {
2773 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
2774 llvm_arg_i += 1;
2775 }
2776
2777 if (fn_info.cc == .async) {
2778 @panic("TODO: LLVM backend lower async function");
2779 }
2780
2781 {
2782 const cc_info = toLlvmCallConv(fn_info.cc, target).?;
2783
2784 function_index.setCallConv(cc_info.llvm_cc, &o.builder);
2785
2786 if (cc_info.align_stack) {
2787 try attributes.addFnAttr(.{ .alignstack = .fromByteUnits(target.stackAlignment()) }, &o.builder);
2788 } else {
2789 _ = try attributes.removeFnAttr(.alignstack);
2790 }
2791
2792 if (cc_info.naked) {
2793 try attributes.addFnAttr(.naked, &o.builder);
2794 } else {
2795 _ = try attributes.removeFnAttr(.naked);
2796 }
2797
2798 for (0..cc_info.inreg_param_count) |param_idx| {
2799 try attributes.addParamAttr(param_idx, .inreg, &o.builder);
2800 }
2801 for (cc_info.inreg_param_count..std.math.maxInt(u2)) |param_idx| {
2802 _ = try attributes.removeParamAttr(param_idx, .inreg);
2803 }
2804
2805 switch (fn_info.cc) {
2806 inline .riscv64_interrupt,
2807 .riscv32_interrupt,
2808 .mips_interrupt,
2809 .mips64_interrupt,
2810 => |info| {
2811 try attributes.addFnAttr(.{ .string = .{
2812 .kind = try o.builder.string("interrupt"),
2813 .value = try o.builder.string(@tagName(info.mode)),
2814 } }, &o.builder);
2815 },
2816 .arm_interrupt,
2817 => |info| {
2818 try attributes.addFnAttr(.{ .string = .{
2819 .kind = try o.builder.string("interrupt"),
2820 .value = try o.builder.string(switch (info.type) {
2821 .generic => "",
2822 .irq => "IRQ",
2823 .fiq => "FIQ",
2824 .swi => "SWI",
2825 .abort => "ABORT",
2826 .undef => "UNDEF",
2827 }),
2828 } }, &o.builder);
2829 },
2830 // these function attributes serve as a backup against any mistakes LLVM makes.
2831 // clang sets both the function's calling convention and the function attributes
2832 // in its backend, so future patches to the AVR backend could end up checking only one,
2833 // possibly breaking our support. it's safer to just emit both.
2834 .avr_interrupt, .avr_signal, .csky_interrupt => {
2835 try attributes.addFnAttr(.{ .string = .{
2836 .kind = try o.builder.string(switch (fn_info.cc) {
2837 .avr_interrupt,
2838 .csky_interrupt,
2839 => "interrupt",
2840 .avr_signal => "signal",
2841 else => unreachable,
2842 }),
2843 .value = .empty,
2844 } }, &o.builder);
2845 },
2846 else => {},
2847 }
2848 }
2849
2850 if (nav.getAlignment() != .none)
2851 function_index.setAlignment(nav.getAlignment().toLlvm(), &o.builder);
2852
2853 // Function attributes that are independent of analysis results of the function body.
2854 try o.addCommonFnAttributes(
2855 &attributes,
2856 owner_mod,
2857 // Some backends don't respect the `naked` attribute in `TargetFrameLowering::hasFP()`,
2858 // so for these backends, LLVM will happily emit code that accesses the stack through
2859 // the frame pointer. This is nonsensical since what the `naked` attribute does is
2860 // suppress generation of the prologue and epilogue, and the prologue is where the
2861 // frame pointer normally gets set up. At time of writing, this is the case for at
2862 // least x86 and RISC-V.
2863 owner_mod.omit_frame_pointer or fn_info.cc == .naked,
2864 );
2865
2866 if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
2867
2868 // Add parameter attributes. We handle only the case of extern functions (no body)
2869 // because functions with bodies are handled in `updateFunc`.
2870 if (is_extern) {
2871 var it = iterateParamTypes(o, pt, fn_info);
2872 it.llvm_index = llvm_arg_i;
2873 while (try it.next()) |lowering| switch (lowering) {
2874 .byval => {
2875 const param_index = it.zig_index - 1;
2876 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
2877 if (!isByRef(param_ty, zcu)) {
2878 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
2879 }
2880 },
2881 .byref => {
2882 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
2883 const param_llvm_ty = try o.lowerType(pt, param_ty);
2884 const alignment = param_ty.abiAlignment(zcu);
2885 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
2886 },
2887 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2888 // No attributes needed for these.
2889 .no_bits,
2890 .abi_sized_int,
2891 .multiple_llvm_types,
2892 .float_array,
2893 .i32_array,
2894 .i64_array,
2895 => continue,
2896
2897 .slice => unreachable, // extern functions do not support slice types.
2898
2899 };
2900 }
2901
2902 function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
2903 return function_index;
2904 }
2905
2906 fn addCommonFnAttributes(
2907 o: *Object,
2908 attributes: *Builder.FunctionAttributes.Wip,
2909 owner_mod: *Package.Module,
2910 omit_frame_pointer: bool,
2911 ) Allocator.Error!void {
2912 if (!owner_mod.red_zone) {
2913 try attributes.addFnAttr(.noredzone, &o.builder);
2914 }
2915 if (omit_frame_pointer) {
2916 try attributes.addFnAttr(.{ .string = .{
2917 .kind = try o.builder.string("frame-pointer"),
2918 .value = try o.builder.string("none"),
2919 } }, &o.builder);
2920 } else {
2921 try attributes.addFnAttr(.{ .string = .{
2922 .kind = try o.builder.string("frame-pointer"),
2923 .value = try o.builder.string("all"),
2924 } }, &o.builder);
2925 }
2926 try attributes.addFnAttr(.nounwind, &o.builder);
2927 if (owner_mod.unwind_tables != .none) {
2928 try attributes.addFnAttr(
2929 .{ .uwtable = if (owner_mod.unwind_tables == .async) .async else .sync },
2930 &o.builder,
2931 );
2932 }
2933 if (owner_mod.optimize_mode == .ReleaseSmall) {
2934 try attributes.addFnAttr(.minsize, &o.builder);
2935 try attributes.addFnAttr(.optsize, &o.builder);
2936 }
2937 const target = &owner_mod.resolved_target.result;
2938 if (target.cpu.model.llvm_name) |s| {
2939 try attributes.addFnAttr(.{ .string = .{
2940 .kind = try o.builder.string("target-cpu"),
2941 .value = try o.builder.string(s),
2942 } }, &o.builder);
2943 }
2944 if (owner_mod.resolved_target.llvm_cpu_features) |s| {
2945 try attributes.addFnAttr(.{ .string = .{
2946 .kind = try o.builder.string("target-features"),
2947 .value = try o.builder.string(std.mem.span(s)),
2948 } }, &o.builder);
2949 }
2950 if (target.abi.float() == .soft) {
2951 // `use-soft-float` means "use software routines for floating point computations". In
2952 // other words, it configures how LLVM lowers basic float instructions like `fcmp`,
2953 // `fadd`, etc. The float calling convention is configured on `TargetMachine` and is
2954 // mostly an orthogonal concept, although obviously we do need hardware float operations
2955 // to actually be able to pass float values in float registers.
2956 //
2957 // Ideally, we would support something akin to the `-mfloat-abi=softfp` option that GCC
2958 // and Clang support for Arm32 and CSKY. We don't currently expose such an option in
2959 // Zig, and using CPU features as the source of truth for this makes for a miserable
2960 // user experience since people expect e.g. `arm-linux-gnueabi` to mean full soft float
2961 // unless the compiler has explicitly been told otherwise. (And note that our baseline
2962 // CPU models almost all include FPU features!)
2963 //
2964 // Revisit this at some point.
2965 try attributes.addFnAttr(.{ .string = .{
2966 .kind = try o.builder.string("use-soft-float"),
2967 .value = try o.builder.string("true"),
2968 } }, &o.builder);
2969
2970 // This prevents LLVM from using FPU/SIMD code for things like `memcpy`. As for the
2971 // above, this should be revisited if `softfp` support is added.
2972 try attributes.addFnAttr(.noimplicitfloat, &o.builder);
2973 }
2974 }
2975
2976 fn resolveGlobalUav(
2977 o: *Object,
2978 pt: Zcu.PerThread,
2979 uav: InternPool.Index,
2980 llvm_addr_space: Builder.AddrSpace,
2981 alignment: InternPool.Alignment,
2982 ) Error!Builder.Variable.Index {
2983 assert(alignment != .none);
2984 // TODO: Add address space to the anon_decl_map
2985 const gop = try o.uav_map.getOrPut(o.gpa, uav);
2986 if (gop.found_existing) {
2987 // Keep the greater of the two alignments.
2988 const variable_index = gop.value_ptr.ptr(&o.builder).kind.variable;
2989 const old_alignment = InternPool.Alignment.fromLlvm(variable_index.getAlignment(&o.builder));
2990 const max_alignment = old_alignment.maxStrict(alignment);
2991 variable_index.setAlignment(max_alignment.toLlvm(), &o.builder);
2992 return variable_index;
2993 }
2994 errdefer assert(o.uav_map.remove(uav));
2995
2996 const zcu = pt.zcu;
2997 const decl_ty = zcu.intern_pool.typeOf(uav);
2998
2999 const variable_index = try o.builder.addVariable(
3000 try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}),
3001 try o.lowerType(pt, Type.fromInterned(decl_ty)),
3002 llvm_addr_space,
3003 );
3004 gop.value_ptr.* = variable_index.ptrConst(&o.builder).global;
3005
3006 try variable_index.setInitializer(try o.lowerValue(pt, uav), &o.builder);
3007 variable_index.setLinkage(.internal, &o.builder);
3008 variable_index.setMutability(.constant, &o.builder);
3009 variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
3010 variable_index.setAlignment(alignment.toLlvm(), &o.builder);
3011 return variable_index;
3012 }
3013
3014 fn resolveGlobalNav(
3015 o: *Object,
3016 pt: Zcu.PerThread,
3017 nav_index: InternPool.Nav.Index,
3018 ) Allocator.Error!Builder.Variable.Index {
3019 const gop = try o.nav_map.getOrPut(o.gpa, nav_index);
3020 if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable;
3021 errdefer assert(o.nav_map.remove(nav_index));
3022
3023 const zcu = pt.zcu;
3024 const ip = &zcu.intern_pool;
3025 const nav = ip.getNav(nav_index);
3026 const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_threadlocal, const is_dll_import = switch (nav.status) {
3027 .unresolved => unreachable,
3028 .fully_resolved => |r| switch (ip.indexToKey(r.val)) {
3029 .variable => |variable| .{ .internal, .default, variable.is_threadlocal, false },
3030 .@"extern" => |@"extern"| .{ @"extern".linkage, .fromSymbolVisibility(@"extern".visibility), @"extern".is_threadlocal, @"extern".is_dll_import },
3031 else => .{ .internal, .default, false, false },
3032 },
3033 // This means it's a source declaration which is not `extern`!
3034 .type_resolved => |r| .{ .internal, .default, r.is_threadlocal, false },
3035 };
3036
3037 const variable_index = try o.builder.addVariable(
3038 try o.builder.strtabString(switch (linkage) {
3039 .internal => nav.fqn,
3040 .strong, .weak => nav.name,
3041 .link_once => unreachable,
3042 }.toSlice(ip)),
3043 try o.lowerType(pt, Type.fromInterned(nav.typeOf(ip))),
3044 toLlvmGlobalAddressSpace(nav.getAddrspace(), zcu.getTarget()),
3045 );
3046 gop.value_ptr.* = variable_index.ptrConst(&o.builder).global;
3047
3048 // This is needed for declarations created by `@extern`.
3049 switch (linkage) {
3050 .internal => {
3051 variable_index.setLinkage(.internal, &o.builder);
3052 variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
3053 },
3054 .strong, .weak => {
3055 variable_index.setLinkage(switch (linkage) {
3056 .internal => unreachable,
3057 .strong => .external,
3058 .weak => .extern_weak,
3059 .link_once => unreachable,
3060 }, &o.builder);
3061 variable_index.setUnnamedAddr(.default, &o.builder);
3062 if (is_threadlocal and !zcu.navFileScope(nav_index).mod.?.single_threaded)
3063 variable_index.setThreadLocal(.generaldynamic, &o.builder);
3064 if (is_dll_import) variable_index.setDllStorageClass(.dllimport, &o.builder);
3065 },
3066 .link_once => unreachable,
3067 }
3068 variable_index.setVisibility(visibility, &o.builder);
3069 return variable_index;
3070 }
3071
3072 fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type {
3073 return o.builder.intType(pt.zcu.errorSetBits());
3074 }
3075
3076 fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type {
3077 const zcu = pt.zcu;
3078 const target = zcu.getTarget();
3079 const ip = &zcu.intern_pool;
3080 return switch (t.toIntern()) {
3081 .u0_type, .i0_type => unreachable,
3082 inline .u1_type,
3083 .u8_type,
3084 .i8_type,
3085 .u16_type,
3086 .i16_type,
3087 .u29_type,
3088 .u32_type,
3089 .i32_type,
3090 .u64_type,
3091 .i64_type,
3092 .u80_type,
3093 .u128_type,
3094 .i128_type,
3095 => |tag| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
3096 .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
3097 inline .c_char_type,
3098 .c_short_type,
3099 .c_ushort_type,
3100 .c_int_type,
3101 .c_uint_type,
3102 .c_long_type,
3103 .c_ulong_type,
3104 .c_longlong_type,
3105 .c_ulonglong_type,
3106 => |tag| try o.builder.intType(target.cTypeBitSize(
3107 @field(std.Target.CType, @tagName(tag)["c_".len .. @tagName(tag).len - "_type".len]),
3108 )),
3109 .c_longdouble_type,
3110 .f16_type,
3111 .f32_type,
3112 .f64_type,
3113 .f80_type,
3114 .f128_type,
3115 => switch (t.floatBits(target)) {
3116 16 => if (backendSupportsF16(target)) .half else .i16,
3117 32 => .float,
3118 64 => .double,
3119 80 => if (backendSupportsF80(target)) .x86_fp80 else .i80,
3120 128 => .fp128,
3121 else => unreachable,
3122 },
3123 .anyopaque_type => {
3124 // This is unreachable except when used as the type for an extern global.
3125 // For example: `@extern(*anyopaque, .{ .name = "foo"})` should produce
3126 // @foo = external global i8
3127 return .i8;
3128 },
3129 .bool_type => .i1,
3130 .void_type => .void,
3131 .type_type => unreachable,
3132 .anyerror_type => try o.errorIntType(pt),
3133 .comptime_int_type,
3134 .comptime_float_type,
3135 .noreturn_type,
3136 => unreachable,
3137 .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
3138 .null_type,
3139 .undefined_type,
3140 .enum_literal_type,
3141 => unreachable,
3142 .ptr_usize_type,
3143 .ptr_const_comptime_int_type,
3144 .manyptr_u8_type,
3145 .manyptr_const_u8_type,
3146 .manyptr_const_u8_sentinel_0_type,
3147 => .ptr,
3148 .slice_const_u8_type,
3149 .slice_const_u8_sentinel_0_type,
3150 => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(pt, Type.usize) }),
3151 .optional_noreturn_type => unreachable,
3152 .anyerror_void_error_union_type,
3153 .adhoc_inferred_error_set_type,
3154 => try o.errorIntType(pt),
3155 .generic_poison_type,
3156 .empty_tuple_type,
3157 => unreachable,
3158 // values, not types
3159 .undef,
3160 .undef_bool,
3161 .undef_usize,
3162 .undef_u1,
3163 .zero,
3164 .zero_usize,
3165 .zero_u1,
3166 .zero_u8,
3167 .one,
3168 .one_usize,
3169 .one_u1,
3170 .one_u8,
3171 .four_u8,
3172 .negative_one,
3173 .void_value,
3174 .unreachable_value,
3175 .null_value,
3176 .bool_true,
3177 .bool_false,
3178 .empty_tuple,
3179 .none,
3180 => unreachable,
3181 else => switch (ip.indexToKey(t.toIntern())) {
3182 .int_type => |int_type| try o.builder.intType(int_type.bits),
3183 .ptr_type => |ptr_type| type: {
3184 const ptr_ty = try o.builder.ptrType(
3185 toLlvmAddressSpace(ptr_type.flags.address_space, target),
3186 );
3187 break :type switch (ptr_type.flags.size) {
3188 .one, .many, .c => ptr_ty,
3189 .slice => try o.builder.structType(.normal, &.{
3190 ptr_ty,
3191 try o.lowerType(pt, Type.usize),
3192 }),
3193 };
3194 },
3195 .array_type => |array_type| o.builder.arrayType(
3196 array_type.lenIncludingSentinel(),
3197 try o.lowerType(pt, Type.fromInterned(array_type.child)),
3198 ),
3199 .vector_type => |vector_type| o.builder.vectorType(
3200 .normal,
3201 vector_type.len,
3202 try o.lowerType(pt, Type.fromInterned(vector_type.child)),
3203 ),
3204 .opt_type => |child_ty| {
3205 // Must stay in sync with `opt_payload` logic in `lowerPtr`.
3206 if (!Type.fromInterned(child_ty).hasRuntimeBitsIgnoreComptime(zcu)) return .i8;
3207
3208 const payload_ty = try o.lowerType(pt, Type.fromInterned(child_ty));
3209 if (t.optionalReprIsPayload(zcu)) return payload_ty;
3210
3211 comptime assert(optional_layout_version == 3);
3212 var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
3213 var fields_len: usize = 2;
3214 const offset = Type.fromInterned(child_ty).abiSize(zcu) + 1;
3215 const abi_size = t.abiSize(zcu);
3216 const padding_len = abi_size - offset;
3217 if (padding_len > 0) {
3218 fields[2] = try o.builder.arrayType(padding_len, .i8);
3219 fields_len = 3;
3220 }
3221 return o.builder.structType(.normal, fields[0..fields_len]);
3222 },
3223 .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
3224 .error_union_type => |error_union_type| {
3225 // Must stay in sync with `codegen.errUnionPayloadOffset`.
3226 // See logic in `lowerPtr`.
3227 const error_type = try o.errorIntType(pt);
3228 if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBitsIgnoreComptime(zcu))
3229 return error_type;
3230 const payload_type = try o.lowerType(pt, Type.fromInterned(error_union_type.payload_type));
3231
3232 const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3233 const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
3234
3235 const payload_size = Type.fromInterned(error_union_type.payload_type).abiSize(zcu);
3236 const error_size = std.zig.target.intByteSize(target, zcu.errorSetBits());
3237
3238 var fields: [3]Builder.Type = undefined;
3239 var fields_len: usize = 2;
3240 const padding_len = if (error_align.compare(.gt, payload_align)) pad: {
3241 fields[0] = error_type;
3242 fields[1] = payload_type;
3243 const payload_end =
3244 payload_align.forward(error_size) +
3245 payload_size;
3246 const abi_size = error_align.forward(payload_end);
3247 break :pad abi_size - payload_end;
3248 } else pad: {
3249 fields[0] = payload_type;
3250 fields[1] = error_type;
3251 const error_end =
3252 error_align.forward(payload_size) +
3253 error_size;
3254 const abi_size = payload_align.forward(error_end);
3255 break :pad abi_size - error_end;
3256 };
3257 if (padding_len > 0) {
3258 fields[2] = try o.builder.arrayType(padding_len, .i8);
3259 fields_len = 3;
3260 }
3261 return o.builder.structType(.normal, fields[0..fields_len]);
3262 },
3263 .simple_type => unreachable,
3264 .struct_type => {
3265 if (o.type_map.get(t.toIntern())) |value| return value;
3266
3267 const struct_type = ip.loadStructType(t.toIntern());
3268
3269 if (struct_type.layout == .@"packed") {
3270 const int_ty = try o.lowerType(pt, Type.fromInterned(struct_type.backingIntTypeUnordered(ip)));
3271 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3272 return int_ty;
3273 }
3274
3275 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
3276 defer llvm_field_types.deinit(o.gpa);
3277 // Although we can estimate how much capacity to add, these cannot be
3278 // relied upon because of the recursive calls to lowerType below.
3279 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3280 try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3281
3282 comptime assert(struct_layout_version == 2);
3283 var offset: u64 = 0;
3284 var big_align: InternPool.Alignment = .@"1";
3285 var struct_kind: Builder.Type.Structure.Kind = .normal;
3286 // When we encounter a zero-bit field, we place it here so we know to map it to the next non-zero-bit field (if any).
3287 var it = struct_type.iterateRuntimeOrder(ip);
3288 while (it.next()) |field_index| {
3289 const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[field_index]);
3290 const field_align = t.fieldAlignment(field_index, zcu);
3291 const field_ty_align = field_ty.abiAlignment(zcu);
3292 if (field_align.compare(.lt, field_ty_align)) struct_kind = .@"packed";
3293 big_align = big_align.max(field_align);
3294 const prev_offset = offset;
3295 offset = field_align.forward(offset);
3296
3297 const padding_len = offset - prev_offset;
3298 if (padding_len > 0) try llvm_field_types.append(
3299 o.gpa,
3300 try o.builder.arrayType(padding_len, .i8),
3301 );
3302
3303 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3304 // This is a zero-bit field. If there are runtime bits after this field,
3305 // map to the next LLVM field (which we know exists): otherwise, don't
3306 // map the field, indicating it's at the end of the struct.
3307 if (offset != struct_type.sizeUnordered(ip)) {
3308 try o.struct_field_map.put(o.gpa, .{
3309 .struct_ty = t.toIntern(),
3310 .field_index = field_index,
3311 }, @intCast(llvm_field_types.items.len));
3312 }
3313 continue;
3314 }
3315
3316 try o.struct_field_map.put(o.gpa, .{
3317 .struct_ty = t.toIntern(),
3318 .field_index = field_index,
3319 }, @intCast(llvm_field_types.items.len));
3320 try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty));
3321
3322 offset += field_ty.abiSize(zcu);
3323 }
3324 {
3325 const prev_offset = offset;
3326 offset = big_align.forward(offset);
3327 const padding_len = offset - prev_offset;
3328 if (padding_len > 0) try llvm_field_types.append(
3329 o.gpa,
3330 try o.builder.arrayType(padding_len, .i8),
3331 );
3332 }
3333
3334 const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3335 try o.type_map.put(o.gpa, t.toIntern(), ty);
3336
3337 o.builder.namedTypeSetBody(
3338 ty,
3339 try o.builder.structType(struct_kind, llvm_field_types.items),
3340 );
3341 return ty;
3342 },
3343 .tuple_type => |tuple_type| {
3344 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
3345 defer llvm_field_types.deinit(o.gpa);
3346 // Although we can estimate how much capacity to add, these cannot be
3347 // relied upon because of the recursive calls to lowerType below.
3348 try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3349 try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3350
3351 comptime assert(struct_layout_version == 2);
3352 var offset: u64 = 0;
3353 var big_align: InternPool.Alignment = .none;
3354
3355 const struct_size = t.abiSize(zcu);
3356
3357 for (
3358 tuple_type.types.get(ip),
3359 tuple_type.values.get(ip),
3360 0..,
3361 ) |field_ty, field_val, field_index| {
3362 if (field_val != .none) continue;
3363
3364 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
3365 big_align = big_align.max(field_align);
3366 const prev_offset = offset;
3367 offset = field_align.forward(offset);
3368
3369 const padding_len = offset - prev_offset;
3370 if (padding_len > 0) try llvm_field_types.append(
3371 o.gpa,
3372 try o.builder.arrayType(padding_len, .i8),
3373 );
3374 if (!Type.fromInterned(field_ty).hasRuntimeBitsIgnoreComptime(zcu)) {
3375 // This is a zero-bit field. If there are runtime bits after this field,
3376 // map to the next LLVM field (which we know exists): otherwise, don't
3377 // map the field, indicating it's at the end of the struct.
3378 if (offset != struct_size) {
3379 try o.struct_field_map.put(o.gpa, .{
3380 .struct_ty = t.toIntern(),
3381 .field_index = @intCast(field_index),
3382 }, @intCast(llvm_field_types.items.len));
3383 }
3384 continue;
3385 }
3386 try o.struct_field_map.put(o.gpa, .{
3387 .struct_ty = t.toIntern(),
3388 .field_index = @intCast(field_index),
3389 }, @intCast(llvm_field_types.items.len));
3390 try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty)));
3391
3392 offset += Type.fromInterned(field_ty).abiSize(zcu);
3393 }
3394 {
3395 const prev_offset = offset;
3396 offset = big_align.forward(offset);
3397 const padding_len = offset - prev_offset;
3398 if (padding_len > 0) try llvm_field_types.append(
3399 o.gpa,
3400 try o.builder.arrayType(padding_len, .i8),
3401 );
3402 }
3403 return o.builder.structType(.normal, llvm_field_types.items);
3404 },
3405 .union_type => {
3406 if (o.type_map.get(t.toIntern())) |value| return value;
3407
3408 const union_obj = ip.loadUnionType(t.toIntern());
3409 const layout = Type.getUnionLayout(union_obj, zcu);
3410
3411 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
3412 const int_ty = try o.builder.intType(@intCast(t.bitSize(zcu)));
3413 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3414 return int_ty;
3415 }
3416
3417 if (layout.payload_size == 0) {
3418 const enum_tag_ty = try o.lowerType(pt, Type.fromInterned(union_obj.enum_tag_ty));
3419 try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3420 return enum_tag_ty;
3421 }
3422
3423 const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3424 const aligned_field_llvm_ty = try o.lowerType(pt, aligned_field_ty);
3425
3426 const payload_ty = ty: {
3427 if (layout.most_aligned_field_size == layout.payload_size) {
3428 break :ty aligned_field_llvm_ty;
3429 }
3430 const padding_len = if (layout.tag_size == 0)
3431 layout.abi_size - layout.most_aligned_field_size
3432 else
3433 layout.payload_size - layout.most_aligned_field_size;
3434 break :ty try o.builder.structType(.@"packed", &.{
3435 aligned_field_llvm_ty,
3436 try o.builder.arrayType(padding_len, .i8),
3437 });
3438 };
3439
3440 if (layout.tag_size == 0) {
3441 const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3442 try o.type_map.put(o.gpa, t.toIntern(), ty);
3443
3444 o.builder.namedTypeSetBody(
3445 ty,
3446 try o.builder.structType(.normal, &.{payload_ty}),
3447 );
3448 return ty;
3449 }
3450 const enum_tag_ty = try o.lowerType(pt, Type.fromInterned(union_obj.enum_tag_ty));
3451
3452 // Put the tag before or after the payload depending on which one's
3453 // alignment is greater.
3454 var llvm_fields: [3]Builder.Type = undefined;
3455 var llvm_fields_len: usize = 2;
3456
3457 if (layout.tag_align.compare(.gte, layout.payload_align)) {
3458 llvm_fields = .{ enum_tag_ty, payload_ty, .none };
3459 } else {
3460 llvm_fields = .{ payload_ty, enum_tag_ty, .none };
3461 }
3462
3463 // Insert padding to make the LLVM struct ABI size match the Zig union ABI size.
3464 if (layout.padding != 0) {
3465 llvm_fields[llvm_fields_len] = try o.builder.arrayType(layout.padding, .i8);
3466 llvm_fields_len += 1;
3467 }
3468
3469 const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3470 try o.type_map.put(o.gpa, t.toIntern(), ty);
3471
3472 o.builder.namedTypeSetBody(
3473 ty,
3474 try o.builder.structType(.normal, llvm_fields[0..llvm_fields_len]),
3475 );
3476 return ty;
3477 },
3478 .opaque_type => {
3479 const gop = try o.type_map.getOrPut(o.gpa, t.toIntern());
3480 if (!gop.found_existing) {
3481 gop.value_ptr.* = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3482 }
3483 return gop.value_ptr.*;
3484 },
3485 .enum_type => try o.lowerType(pt, Type.fromInterned(ip.loadEnumType(t.toIntern()).tag_ty)),
3486 .func_type => |func_type| try o.lowerTypeFn(pt, func_type),
3487 .error_set_type, .inferred_error_set_type => try o.errorIntType(pt),
3488 // values, not types
3489 .undef,
3490 .simple_value,
3491 .variable,
3492 .@"extern",
3493 .func,
3494 .int,
3495 .err,
3496 .error_union,
3497 .enum_literal,
3498 .enum_tag,
3499 .empty_enum_value,
3500 .float,
3501 .ptr,
3502 .slice,
3503 .opt,
3504 .aggregate,
3505 .un,
3506 // memoization, not types
3507 .memoized_call,
3508 => unreachable,
3509 },
3510 };
3511 }
3512
3513 /// Use this instead of lowerType when you want to handle correctly the case of elem_ty
3514 /// being a zero bit type, but it should still be lowered as an i8 in such case.
3515 /// There are other similar cases handled here as well.
3516 fn lowerPtrElemTy(o: *Object, pt: Zcu.PerThread, elem_ty: Type) Allocator.Error!Builder.Type {
3517 const zcu = pt.zcu;
3518 const lower_elem_ty = switch (elem_ty.zigTypeTag(zcu)) {
3519 .@"opaque" => true,
3520 .@"fn" => !zcu.typeToFunc(elem_ty).?.is_generic,
3521 .array => elem_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu),
3522 else => elem_ty.hasRuntimeBitsIgnoreComptime(zcu),
3523 };
3524 return if (lower_elem_ty) try o.lowerType(pt, elem_ty) else .i8;
3525 }
3526
3527 fn lowerTypeFn(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
3528 const zcu = pt.zcu;
3529 const ip = &zcu.intern_pool;
3530 const target = zcu.getTarget();
3531 const ret_ty = try lowerFnRetTy(o, pt, fn_info);
3532
3533 var llvm_params: std.ArrayList(Builder.Type) = .empty;
3534 defer llvm_params.deinit(o.gpa);
3535
3536 if (firstParamSRet(fn_info, zcu, target)) {
3537 try llvm_params.append(o.gpa, .ptr);
3538 }
3539
3540 if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
3541 const stack_trace_ty = zcu.builtin_decl_values.get(.StackTrace);
3542 const ptr_ty = try pt.ptrType(.{ .child = stack_trace_ty });
3543 try llvm_params.append(o.gpa, try o.lowerType(pt, ptr_ty));
3544 }
3545
3546 var it = iterateParamTypes(o, pt, fn_info);
3547 while (try it.next()) |lowering| switch (lowering) {
3548 .no_bits => continue,
3549 .byval => {
3550 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3551 try llvm_params.append(o.gpa, try o.lowerType(pt, param_ty));
3552 },
3553 .byref, .byref_mut => {
3554 try llvm_params.append(o.gpa, .ptr);
3555 },
3556 .abi_sized_int => {
3557 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3558 try llvm_params.append(o.gpa, try o.builder.intType(
3559 @intCast(param_ty.abiSize(zcu) * 8),
3560 ));
3561 },
3562 .slice => {
3563 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3564 try llvm_params.appendSlice(o.gpa, &.{
3565 try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3566 try o.lowerType(pt, Type.usize),
3567 });
3568 },
3569 .multiple_llvm_types => {
3570 try llvm_params.appendSlice(o.gpa, it.types_buffer[0..it.types_len]);
3571 },
3572 .float_array => |count| {
3573 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3574 const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
3575 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3576 },
3577 .i32_array, .i64_array => |arr_len| {
3578 try llvm_params.append(o.gpa, try o.builder.arrayType(arr_len, switch (lowering) {
3579 .i32_array => .i32,
3580 .i64_array => .i64,
3581 else => unreachable,
3582 }));
3583 },
3584 };
3585
3586 return o.builder.fnType(
3587 ret_ty,
3588 llvm_params.items,
3589 if (fn_info.is_var_args) .vararg else .normal,
3590 );
3591 }
3592
3593 fn lowerValueToInt(o: *Object, pt: Zcu.PerThread, llvm_int_ty: Builder.Type, arg_val: InternPool.Index) Error!Builder.Constant {
3594 const zcu = pt.zcu;
3595 const ip = &zcu.intern_pool;
3596 const target = zcu.getTarget();
3597
3598 const val = Value.fromInterned(arg_val);
3599 const val_key = ip.indexToKey(val.toIntern());
3600
3601 if (val.isUndef(zcu)) return o.builder.undefConst(llvm_int_ty);
3602
3603 const ty = Type.fromInterned(val_key.typeOf());
3604 switch (val_key) {
3605 .@"extern" => |@"extern"| {
3606 const function_index = try o.resolveLlvmFunction(pt, @"extern".owner_nav);
3607 const ptr = function_index.ptrConst(&o.builder).global.toConst();
3608 return o.builder.convConst(ptr, llvm_int_ty);
3609 },
3610 .func => |func| {
3611 const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
3612 const ptr = function_index.ptrConst(&o.builder).global.toConst();
3613 return o.builder.convConst(ptr, llvm_int_ty);
3614 },
3615 .ptr => return o.builder.convConst(try o.lowerPtr(pt, arg_val, 0), llvm_int_ty),
3616 .aggregate => switch (ip.indexToKey(ty.toIntern())) {
3617 .struct_type, .vector_type => {},
3618 else => unreachable,
3619 },
3620 .un => |un| {
3621 const layout = ty.unionGetLayout(zcu);
3622 if (layout.payload_size == 0) return o.lowerValue(pt, un.tag);
3623
3624 const union_obj = zcu.typeToUnion(ty).?;
3625 const container_layout = union_obj.flagsUnordered(ip).layout;
3626
3627 assert(container_layout == .@"packed");
3628
3629 var need_unnamed = false;
3630 if (un.tag == .none) {
3631 assert(layout.tag_size == 0);
3632 const union_val = try o.lowerValueToInt(pt, llvm_int_ty, un.val);
3633
3634 need_unnamed = true;
3635 return union_val;
3636 }
3637 const field_index = zcu.unionTagFieldIndex(union_obj, Value.fromInterned(un.tag)).?;
3638 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[field_index]);
3639 if (!field_ty.hasRuntimeBits(zcu)) return o.builder.intConst(llvm_int_ty, 0);
3640 return o.lowerValueToInt(pt, llvm_int_ty, un.val);
3641 },
3642 .simple_value => |simple_value| switch (simple_value) {
3643 .false, .true => {},
3644 else => unreachable,
3645 },
3646 .int,
3647 .float,
3648 .enum_tag,
3649 => {},
3650 .opt => {}, // pointer like optional expected
3651 else => unreachable,
3652 }
3653 var stack = std.heap.stackFallback(32, o.gpa);
3654 const allocator = stack.get();
3655
3656 const bits: usize = @intCast(ty.bitSize(zcu));
3657
3658 const buffer = try allocator.alloc(u8, (bits + 7) / 8);
3659 defer allocator.free(buffer);
3660 const limbs = try allocator.alloc(std.math.big.Limb, std.math.big.int.calcTwosCompLimbCount(bits));
3661 defer allocator.free(limbs);
3662
3663 val.writeToPackedMemory(ty, pt, buffer, 0) catch unreachable;
3664
3665 var big: std.math.big.int.Mutable = .init(limbs, 0);
3666 big.readTwosComplement(buffer, bits, target.cpu.arch.endian(), .unsigned);
3667
3668 return o.builder.bigIntConst(llvm_int_ty, big.toConst());
3669 }
3670
3671 fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Error!Builder.Constant {
3672 const zcu = pt.zcu;
3673 const ip = &zcu.intern_pool;
3674 const target = zcu.getTarget();
3675
3676 const val = Value.fromInterned(arg_val);
3677 const val_key = ip.indexToKey(val.toIntern());
3678
3679 if (val.isUndef(zcu)) {
3680 return o.builder.undefConst(try o.lowerType(pt, Type.fromInterned(val_key.typeOf())));
3681 }
3682
3683 const ty = Type.fromInterned(val_key.typeOf());
3684 return switch (val_key) {
3685 .int_type,
3686 .ptr_type,
3687 .array_type,
3688 .vector_type,
3689 .opt_type,
3690 .anyframe_type,
3691 .error_union_type,
3692 .simple_type,
3693 .struct_type,
3694 .tuple_type,
3695 .union_type,
3696 .opaque_type,
3697 .enum_type,
3698 .func_type,
3699 .error_set_type,
3700 .inferred_error_set_type,
3701 => unreachable, // types, not values
3702
3703 .undef => unreachable, // handled above
3704 .simple_value => |simple_value| switch (simple_value) {
3705 .undefined => unreachable, // non-runtime value
3706 .void => unreachable, // non-runtime value
3707 .null => unreachable, // non-runtime value
3708 .empty_tuple => unreachable, // non-runtime value
3709 .@"unreachable" => unreachable, // non-runtime value
3710
3711 .false => .false,
3712 .true => .true,
3713 },
3714 .variable,
3715 .enum_literal,
3716 .empty_enum_value,
3717 => unreachable, // non-runtime values
3718 .@"extern" => |@"extern"| {
3719 const function_index = try o.resolveLlvmFunction(pt, @"extern".owner_nav);
3720 return function_index.ptrConst(&o.builder).global.toConst();
3721 },
3722 .func => |func| {
3723 const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
3724 return function_index.ptrConst(&o.builder).global.toConst();
3725 },
3726 .int => {
3727 var bigint_space: Value.BigIntSpace = undefined;
3728 const bigint = val.toBigInt(&bigint_space, zcu);
3729 return lowerBigInt(o, pt, ty, bigint);
3730 },
3731 .err => |err| {
3732 const int = try pt.getErrorValue(err.name);
3733 const llvm_int = try o.builder.intConst(try o.errorIntType(pt), int);
3734 return llvm_int;
3735 },
3736 .error_union => |error_union| {
3737 const err_val = switch (error_union.val) {
3738 .err_name => |err_name| try pt.intern(.{ .err = .{
3739 .ty = ty.errorUnionSet(zcu).toIntern(),
3740 .name = err_name,
3741 } }),
3742 .payload => (try pt.intValue(try pt.errorIntType(), 0)).toIntern(),
3743 };
3744 const err_int_ty = try pt.errorIntType();
3745 const payload_type = ty.errorUnionPayload(zcu);
3746 if (!payload_type.hasRuntimeBitsIgnoreComptime(zcu)) {
3747 // We use the error type directly as the type.
3748 return o.lowerValue(pt, err_val);
3749 }
3750
3751 const payload_align = payload_type.abiAlignment(zcu);
3752 const error_align = err_int_ty.abiAlignment(zcu);
3753 const llvm_error_value = try o.lowerValue(pt, err_val);
3754 const llvm_payload_value = try o.lowerValue(pt, switch (error_union.val) {
3755 .err_name => try pt.intern(.{ .undef = payload_type.toIntern() }),
3756 .payload => |payload| payload,
3757 });
3758
3759 var fields: [3]Builder.Type = undefined;
3760 var vals: [3]Builder.Constant = undefined;
3761 if (error_align.compare(.gt, payload_align)) {
3762 vals[0] = llvm_error_value;
3763 vals[1] = llvm_payload_value;
3764 } else {
3765 vals[0] = llvm_payload_value;
3766 vals[1] = llvm_error_value;
3767 }
3768 fields[0] = vals[0].typeOf(&o.builder);
3769 fields[1] = vals[1].typeOf(&o.builder);
3770
3771 const llvm_ty = try o.lowerType(pt, ty);
3772 const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3773 if (llvm_ty_fields.len > 2) {
3774 assert(llvm_ty_fields.len == 3);
3775 fields[2] = llvm_ty_fields[2];
3776 vals[2] = try o.builder.undefConst(fields[2]);
3777 }
3778 return o.builder.structConst(try o.builder.structType(
3779 llvm_ty.structKind(&o.builder),
3780 fields[0..llvm_ty_fields.len],
3781 ), vals[0..llvm_ty_fields.len]);
3782 },
3783 .enum_tag => |enum_tag| o.lowerValue(pt, enum_tag.int),
3784 .float => switch (ty.floatBits(target)) {
3785 16 => if (backendSupportsF16(target))
3786 try o.builder.halfConst(val.toFloat(f16, zcu))
3787 else
3788 try o.builder.intConst(.i16, @as(i16, @bitCast(val.toFloat(f16, zcu)))),
3789 32 => try o.builder.floatConst(val.toFloat(f32, zcu)),
3790 64 => try o.builder.doubleConst(val.toFloat(f64, zcu)),
3791 80 => if (backendSupportsF80(target))
3792 try o.builder.x86_fp80Const(val.toFloat(f80, zcu))
3793 else
3794 try o.builder.intConst(.i80, @as(i80, @bitCast(val.toFloat(f80, zcu)))),
3795 128 => try o.builder.fp128Const(val.toFloat(f128, zcu)),
3796 else => unreachable,
3797 },
3798 .ptr => try o.lowerPtr(pt, arg_val, 0),
3799 .slice => |slice| return o.builder.structConst(try o.lowerType(pt, ty), &.{
3800 try o.lowerValue(pt, slice.ptr),
3801 try o.lowerValue(pt, slice.len),
3802 }),
3803 .opt => |opt| {
3804 comptime assert(optional_layout_version == 3);
3805 const payload_ty = ty.optionalChild(zcu);
3806
3807 const non_null_bit = try o.builder.intConst(.i8, @intFromBool(opt.val != .none));
3808 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3809 return non_null_bit;
3810 }
3811 const llvm_ty = try o.lowerType(pt, ty);
3812 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3813 .none => switch (llvm_ty.tag(&o.builder)) {
3814 .integer => try o.builder.intConst(llvm_ty, 0),
3815 .pointer => try o.builder.nullConst(llvm_ty),
3816 .structure => try o.builder.zeroInitConst(llvm_ty),
3817 else => unreachable,
3818 },
3819 else => |payload| try o.lowerValue(pt, payload),
3820 };
3821 assert(payload_ty.zigTypeTag(zcu) != .@"fn");
3822
3823 var fields: [3]Builder.Type = undefined;
3824 var vals: [3]Builder.Constant = undefined;
3825 vals[0] = try o.lowerValue(pt, switch (opt.val) {
3826 .none => try pt.intern(.{ .undef = payload_ty.toIntern() }),
3827 else => |payload| payload,
3828 });
3829 vals[1] = non_null_bit;
3830 fields[0] = vals[0].typeOf(&o.builder);
3831 fields[1] = vals[1].typeOf(&o.builder);
3832
3833 const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3834 if (llvm_ty_fields.len > 2) {
3835 assert(llvm_ty_fields.len == 3);
3836 fields[2] = llvm_ty_fields[2];
3837 vals[2] = try o.builder.undefConst(fields[2]);
3838 }
3839 return o.builder.structConst(try o.builder.structType(
3840 llvm_ty.structKind(&o.builder),
3841 fields[0..llvm_ty_fields.len],
3842 ), vals[0..llvm_ty_fields.len]);
3843 },
3844 .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {
3845 .array_type => |array_type| switch (aggregate.storage) {
3846 .bytes => |bytes| try o.builder.stringConst(try o.builder.string(
3847 bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3848 )),
3849 .elems => |elems| {
3850 const array_ty = try o.lowerType(pt, ty);
3851 const elem_ty = array_ty.childType(&o.builder);
3852 assert(elems.len == array_ty.aggregateLen(&o.builder));
3853
3854 const ExpectedContents = extern struct {
3855 vals: [Builder.expected_fields_len]Builder.Constant,
3856 fields: [Builder.expected_fields_len]Builder.Type,
3857 };
3858 var stack align(@max(
3859 @alignOf(std.heap.StackFallbackAllocator(0)),
3860 @alignOf(ExpectedContents),
3861 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), o.gpa);
3862 const allocator = stack.get();
3863 const vals = try allocator.alloc(Builder.Constant, elems.len);
3864 defer allocator.free(vals);
3865 const fields = try allocator.alloc(Builder.Type, elems.len);
3866 defer allocator.free(fields);
3867
3868 var need_unnamed = false;
3869 for (vals, fields, elems) |*result_val, *result_field, elem| {
3870 result_val.* = try o.lowerValue(pt, elem);
3871 result_field.* = result_val.typeOf(&o.builder);
3872 if (result_field.* != elem_ty) need_unnamed = true;
3873 }
3874 return if (need_unnamed) try o.builder.structConst(
3875 try o.builder.structType(.normal, fields),
3876 vals,
3877 ) else try o.builder.arrayConst(array_ty, vals);
3878 },
3879 .repeated_elem => |elem| {
3880 const len: usize = @intCast(array_type.len);
3881 const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3882 const array_ty = try o.lowerType(pt, ty);
3883 const elem_ty = array_ty.childType(&o.builder);
3884
3885 const ExpectedContents = extern struct {
3886 vals: [Builder.expected_fields_len]Builder.Constant,
3887 fields: [Builder.expected_fields_len]Builder.Type,
3888 };
3889 var stack align(@max(
3890 @alignOf(std.heap.StackFallbackAllocator(0)),
3891 @alignOf(ExpectedContents),
3892 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), o.gpa);
3893 const allocator = stack.get();
3894 const vals = try allocator.alloc(Builder.Constant, len_including_sentinel);
3895 defer allocator.free(vals);
3896 const fields = try allocator.alloc(Builder.Type, len_including_sentinel);
3897 defer allocator.free(fields);
3898
3899 var need_unnamed = false;
3900 @memset(vals[0..len], try o.lowerValue(pt, elem));
3901 @memset(fields[0..len], vals[0].typeOf(&o.builder));
3902 if (fields[0] != elem_ty) need_unnamed = true;
3903
3904 if (array_type.sentinel != .none) {
3905 vals[len] = try o.lowerValue(pt, array_type.sentinel);
3906 fields[len] = vals[len].typeOf(&o.builder);
3907 if (fields[len] != elem_ty) need_unnamed = true;
3908 }
3909
3910 return if (need_unnamed) try o.builder.structConst(
3911 try o.builder.structType(.@"packed", fields),
3912 vals,
3913 ) else try o.builder.arrayConst(array_ty, vals);
3914 },
3915 },
3916 .vector_type => |vector_type| {
3917 const vector_ty = try o.lowerType(pt, ty);
3918 switch (aggregate.storage) {
3919 .bytes, .elems => {
3920 const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
3921 var stack align(@max(
3922 @alignOf(std.heap.StackFallbackAllocator(0)),
3923 @alignOf(ExpectedContents),
3924 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), o.gpa);
3925 const allocator = stack.get();
3926 const vals = try allocator.alloc(Builder.Constant, vector_type.len);
3927 defer allocator.free(vals);
3928
3929 switch (aggregate.storage) {
3930 .bytes => |bytes| for (vals, bytes.toSlice(vector_type.len, ip)) |*result_val, byte| {
3931 result_val.* = try o.builder.intConst(.i8, byte);
3932 },
3933 .elems => |elems| for (vals, elems) |*result_val, elem| {
3934 result_val.* = try o.lowerValue(pt, elem);
3935 },
3936 .repeated_elem => unreachable,
3937 }
3938 return o.builder.vectorConst(vector_ty, vals);
3939 },
3940 .repeated_elem => |elem| return o.builder.splatConst(
3941 vector_ty,
3942 try o.lowerValue(pt, elem),
3943 ),
3944 }
3945 },
3946 .tuple_type => |tuple| {
3947 const struct_ty = try o.lowerType(pt, ty);
3948 const llvm_len = struct_ty.aggregateLen(&o.builder);
3949
3950 const ExpectedContents = extern struct {
3951 vals: [Builder.expected_fields_len]Builder.Constant,
3952 fields: [Builder.expected_fields_len]Builder.Type,
3953 };
3954 var stack align(@max(
3955 @alignOf(std.heap.StackFallbackAllocator(0)),
3956 @alignOf(ExpectedContents),
3957 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), o.gpa);
3958 const allocator = stack.get();
3959 const vals = try allocator.alloc(Builder.Constant, llvm_len);
3960 defer allocator.free(vals);
3961 const fields = try allocator.alloc(Builder.Type, llvm_len);
3962 defer allocator.free(fields);
3963
3964 comptime assert(struct_layout_version == 2);
3965 var llvm_index: usize = 0;
3966 var offset: u64 = 0;
3967 var big_align: InternPool.Alignment = .none;
3968 var need_unnamed = false;
3969 for (
3970 tuple.types.get(ip),
3971 tuple.values.get(ip),
3972 0..,
3973 ) |field_ty, field_val, field_index| {
3974 if (field_val != .none) continue;
3975 if (!Type.fromInterned(field_ty).hasRuntimeBitsIgnoreComptime(zcu)) continue;
3976
3977 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
3978 big_align = big_align.max(field_align);
3979 const prev_offset = offset;
3980 offset = field_align.forward(offset);
3981
3982 const padding_len = offset - prev_offset;
3983 if (padding_len > 0) {
3984 // TODO make this and all other padding elsewhere in debug
3985 // builds be 0xaa not undef.
3986 fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
3987 vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
3988 assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
3989 llvm_index += 1;
3990 }
3991
3992 vals[llvm_index] =
3993 try o.lowerValue(pt, (try val.fieldValue(pt, field_index)).toIntern());
3994 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3995 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
3996 need_unnamed = true;
3997 llvm_index += 1;
3998
3999 offset += Type.fromInterned(field_ty).abiSize(zcu);
4000 }
4001 {
4002 const prev_offset = offset;
4003 offset = big_align.forward(offset);
4004 const padding_len = offset - prev_offset;
4005 if (padding_len > 0) {
4006 fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
4007 vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
4008 assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
4009 llvm_index += 1;
4010 }
4011 }
4012 assert(llvm_index == llvm_len);
4013
4014 return o.builder.structConst(if (need_unnamed)
4015 try o.builder.structType(struct_ty.structKind(&o.builder), fields)
4016 else
4017 struct_ty, vals);
4018 },
4019 .struct_type => {
4020 const struct_type = ip.loadStructType(ty.toIntern());
4021 assert(struct_type.haveLayout(ip));
4022 const struct_ty = try o.lowerType(pt, ty);
4023 if (struct_type.layout == .@"packed") {
4024 comptime assert(Type.packed_struct_layout_version == 2);
4025
4026 const bits = ty.bitSize(zcu);
4027 const llvm_int_ty = try o.builder.intType(@intCast(bits));
4028
4029 return o.lowerValueToInt(pt, llvm_int_ty, arg_val);
4030 }
4031 const llvm_len = struct_ty.aggregateLen(&o.builder);
4032
4033 const ExpectedContents = extern struct {
4034 vals: [Builder.expected_fields_len]Builder.Constant,
4035 fields: [Builder.expected_fields_len]Builder.Type,
4036 };
4037 var stack align(@max(
4038 @alignOf(std.heap.StackFallbackAllocator(0)),
4039 @alignOf(ExpectedContents),
4040 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), o.gpa);
4041 const allocator = stack.get();
4042 const vals = try allocator.alloc(Builder.Constant, llvm_len);
4043 defer allocator.free(vals);
4044 const fields = try allocator.alloc(Builder.Type, llvm_len);
4045 defer allocator.free(fields);
4046
4047 comptime assert(struct_layout_version == 2);
4048 var llvm_index: usize = 0;
4049 var offset: u64 = 0;
4050 var big_align: InternPool.Alignment = .@"1";
4051 var need_unnamed = false;
4052 var field_it = struct_type.iterateRuntimeOrder(ip);
4053 while (field_it.next()) |field_index| {
4054 const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[field_index]);
4055 const field_align = ty.fieldAlignment(field_index, zcu);
4056 big_align = big_align.max(field_align);
4057 const prev_offset = offset;
4058 offset = field_align.forward(offset);
4059
4060 const padding_len = offset - prev_offset;
4061 if (padding_len > 0) {
4062 // TODO make this and all other padding elsewhere in debug
4063 // builds be 0xaa not undef.
4064 fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
4065 vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
4066 assert(fields[llvm_index] ==
4067 struct_ty.structFields(&o.builder)[llvm_index]);
4068 llvm_index += 1;
4069 }
4070
4071 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4072 // This is a zero-bit field - we only needed it for the alignment.
4073 continue;
4074 }
4075
4076 vals[llvm_index] = try o.lowerValue(
4077 pt,
4078 (try val.fieldValue(pt, field_index)).toIntern(),
4079 );
4080 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
4081 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
4082 need_unnamed = true;
4083 llvm_index += 1;
4084
4085 offset += field_ty.abiSize(zcu);
4086 }
4087 {
4088 const prev_offset = offset;
4089 offset = big_align.forward(offset);
4090 const padding_len = offset - prev_offset;
4091 if (padding_len > 0) {
4092 fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
4093 vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
4094 assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
4095 llvm_index += 1;
4096 }
4097 }
4098 assert(llvm_index == llvm_len);
4099
4100 return o.builder.structConst(if (need_unnamed)
4101 try o.builder.structType(struct_ty.structKind(&o.builder), fields)
4102 else
4103 struct_ty, vals);
4104 },
4105 else => unreachable,
4106 },
4107 .un => |un| {
4108 const union_ty = try o.lowerType(pt, ty);
4109 const layout = ty.unionGetLayout(zcu);
4110 if (layout.payload_size == 0) return o.lowerValue(pt, un.tag);
4111
4112 const union_obj = zcu.typeToUnion(ty).?;
4113 const container_layout = union_obj.flagsUnordered(ip).layout;
4114
4115 var need_unnamed = false;
4116 const payload = if (un.tag != .none) p: {
4117 const field_index = zcu.unionTagFieldIndex(union_obj, Value.fromInterned(un.tag)).?;
4118 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[field_index]);
4119 if (container_layout == .@"packed") {
4120 if (!field_ty.hasRuntimeBits(zcu)) return o.builder.intConst(union_ty, 0);
4121 const bits = ty.bitSize(zcu);
4122 const llvm_int_ty = try o.builder.intType(@intCast(bits));
4123
4124 return o.lowerValueToInt(pt, llvm_int_ty, arg_val);
4125 }
4126
4127 // Sometimes we must make an unnamed struct because LLVM does
4128 // not support bitcasting our payload struct to the true union payload type.
4129 // Instead we use an unnamed struct and every reference to the global
4130 // must pointer cast to the expected type before accessing the union.
4131 need_unnamed = layout.most_aligned_field != field_index;
4132
4133 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
4134 const padding_len = layout.payload_size;
4135 break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
4136 }
4137 const payload = try o.lowerValue(pt, un.val);
4138 const payload_ty = payload.typeOf(&o.builder);
4139 if (payload_ty != union_ty.structFields(&o.builder)[
4140 @intFromBool(layout.tag_align.compare(.gte, layout.payload_align))
4141 ]) need_unnamed = true;
4142 const field_size = field_ty.abiSize(zcu);
4143 if (field_size == layout.payload_size) break :p payload;
4144 const padding_len = layout.payload_size - field_size;
4145 const padding_ty = try o.builder.arrayType(padding_len, .i8);
4146 break :p try o.builder.structConst(
4147 try o.builder.structType(.@"packed", &.{ payload_ty, padding_ty }),
4148 &.{ payload, try o.builder.undefConst(padding_ty) },
4149 );
4150 } else p: {
4151 assert(layout.tag_size == 0);
4152 if (container_layout == .@"packed") {
4153 const bits = ty.bitSize(zcu);
4154 const llvm_int_ty = try o.builder.intType(@intCast(bits));
4155
4156 return o.lowerValueToInt(pt, llvm_int_ty, arg_val);
4157 }
4158
4159 const union_val = try o.lowerValue(pt, un.val);
4160 need_unnamed = true;
4161 break :p union_val;
4162 };
4163
4164 const payload_ty = payload.typeOf(&o.builder);
4165 if (layout.tag_size == 0) return o.builder.structConst(if (need_unnamed)
4166 try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
4167 else
4168 union_ty, &.{payload});
4169 const tag = try o.lowerValue(pt, un.tag);
4170 const tag_ty = tag.typeOf(&o.builder);
4171 var fields: [3]Builder.Type = undefined;
4172 var vals: [3]Builder.Constant = undefined;
4173 var len: usize = 2;
4174 if (layout.tag_align.compare(.gte, layout.payload_align)) {
4175 fields = .{ tag_ty, payload_ty, undefined };
4176 vals = .{ tag, payload, undefined };
4177 } else {
4178 fields = .{ payload_ty, tag_ty, undefined };
4179 vals = .{ payload, tag, undefined };
4180 }
4181 if (layout.padding != 0) {
4182 fields[2] = try o.builder.arrayType(layout.padding, .i8);
4183 vals[2] = try o.builder.undefConst(fields[2]);
4184 len = 3;
4185 }
4186 return o.builder.structConst(if (need_unnamed)
4187 try o.builder.structType(union_ty.structKind(&o.builder), fields[0..len])
4188 else
4189 union_ty, vals[0..len]);
4190 },
4191 .memoized_call => unreachable,
4192 };
4193 }
4194
4195 fn lowerBigInt(
4196 o: *Object,
4197 pt: Zcu.PerThread,
4198 ty: Type,
4199 bigint: std.math.big.int.Const,
4200 ) Allocator.Error!Builder.Constant {
4201 const zcu = pt.zcu;
4202 return o.builder.bigIntConst(try o.builder.intType(ty.intInfo(zcu).bits), bigint);
4203 }
4204
4205 fn lowerPtr(
4206 o: *Object,
4207 pt: Zcu.PerThread,
4208 ptr_val: InternPool.Index,
4209 prev_offset: u64,
4210 ) Error!Builder.Constant {
4211 const zcu = pt.zcu;
4212 const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
4213 const offset: u64 = prev_offset + ptr.byte_offset;
4214 return switch (ptr.base_addr) {
4215 .nav => |nav| {
4216 const base_ptr = try o.lowerNavRefValue(pt, nav);
4217 return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
4218 try o.builder.intConst(.i64, offset),
4219 });
4220 },
4221 .uav => |uav| {
4222 const base_ptr = try o.lowerUavRef(pt, uav);
4223 return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
4224 try o.builder.intConst(.i64, offset),
4225 });
4226 },
4227 .int => try o.builder.castConst(
4228 .inttoptr,
4229 try o.builder.intConst(try o.lowerType(pt, Type.usize), offset),
4230 try o.lowerType(pt, Type.fromInterned(ptr.ty)),
4231 ),
4232 .eu_payload => |eu_ptr| try o.lowerPtr(
4233 pt,
4234 eu_ptr,
4235 offset + codegen.errUnionPayloadOffset(
4236 Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
4237 zcu,
4238 ),
4239 ),
4240 .opt_payload => |opt_ptr| try o.lowerPtr(pt, opt_ptr, offset),
4241 .field => |field| {
4242 const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu);
4243 const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) {
4244 .pointer => off: {
4245 assert(agg_ty.isSlice(zcu));
4246 break :off switch (field.index) {
4247 Value.slice_ptr_index => 0,
4248 Value.slice_len_index => @divExact(zcu.getTarget().ptrBitWidth(), 8),
4249 else => unreachable,
4250 };
4251 },
4252 .@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {
4253 .auto => agg_ty.structFieldOffset(@intCast(field.index), zcu),
4254 .@"extern", .@"packed" => unreachable,
4255 },
4256 else => unreachable,
4257 };
4258 return o.lowerPtr(pt, field.base, offset + field_off);
4259 },
4260 .arr_elem, .comptime_field, .comptime_alloc => unreachable,
4261 };
4262 }
4263
4264 /// This logic is very similar to `lowerNavRefValue` but for anonymous declarations.
4265 /// Maybe the logic could be unified.
4266 fn lowerUavRef(
4267 o: *Object,
4268 pt: Zcu.PerThread,
4269 uav: InternPool.Key.Ptr.BaseAddr.Uav,
4270 ) Error!Builder.Constant {
4271 const zcu = pt.zcu;
4272 const ip = &zcu.intern_pool;
4273 const uav_val = uav.val;
4274 const uav_ty = Type.fromInterned(ip.typeOf(uav_val));
4275 const target = zcu.getTarget();
4276
4277 switch (ip.indexToKey(uav_val)) {
4278 .func => @panic("TODO"),
4279 .@"extern" => @panic("TODO"),
4280 else => {},
4281 }
4282
4283 const ptr_ty = Type.fromInterned(uav.orig_ty);
4284
4285 const is_fn_body = uav_ty.zigTypeTag(zcu) == .@"fn";
4286 if ((!is_fn_body and !uav_ty.hasRuntimeBits(zcu)) or
4287 (is_fn_body and zcu.typeToFunc(uav_ty).?.is_generic)) return o.lowerPtrToVoid(pt, ptr_ty);
4288
4289 if (is_fn_body)
4290 @panic("TODO");
4291
4292 const llvm_addr_space = toLlvmAddressSpace(ptr_ty.ptrAddressSpace(zcu), target);
4293 const alignment = ptr_ty.ptrAlignment(zcu);
4294 const llvm_global = (try o.resolveGlobalUav(pt, uav.val, llvm_addr_space, alignment)).ptrConst(&o.builder).global;
4295
4296 const llvm_val = try o.builder.convConst(
4297 llvm_global.toConst(),
4298 try o.builder.ptrType(llvm_addr_space),
4299 );
4300
4301 return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4302 }
4303
4304 fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant {
4305 const zcu = pt.zcu;
4306 const ip = &zcu.intern_pool;
4307
4308 const nav = ip.getNav(nav_index);
4309
4310 const nav_ty = Type.fromInterned(nav.typeOf(ip));
4311 const ptr_ty = try pt.navPtrType(nav_index);
4312
4313 const is_fn_body = nav_ty.zigTypeTag(zcu) == .@"fn";
4314 if ((!is_fn_body and !nav_ty.hasRuntimeBits(zcu)) or
4315 (is_fn_body and zcu.typeToFunc(nav_ty).?.is_generic))
4316 {
4317 return o.lowerPtrToVoid(pt, ptr_ty);
4318 }
4319
4320 const llvm_global = if (is_fn_body)
4321 (try o.resolveLlvmFunction(pt, nav_index)).ptrConst(&o.builder).global
4322 else
4323 (try o.resolveGlobalNav(pt, nav_index)).ptrConst(&o.builder).global;
4324
4325 const llvm_val = try o.builder.convConst(
4326 llvm_global.toConst(),
4327 try o.builder.ptrType(toLlvmAddressSpace(nav.getAddrspace(), zcu.getTarget())),
4328 );
4329
4330 return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4331 }
4332
4333 fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant {
4334 const zcu = pt.zcu;
4335 // Even though we are pointing at something which has zero bits (e.g. `void`),
4336 // Pointers are defined to have bits. So we must return something here.
4337 // The value cannot be undefined, because we use the `nonnull` annotation
4338 // for non-optional pointers. We also need to respect the alignment, even though
4339 // the address will never be dereferenced.
4340 const int: u64 = ptr_ty.ptrInfo(zcu).flags.alignment.toByteUnits() orelse
4341 // Note that these 0xaa values are appropriate even in release-optimized builds
4342 // because we need a well-defined value that is not null, and LLVM does not
4343 // have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR
4344 // instruction is followed by a `wrap_optional`, it will return this value
4345 // verbatim, and the result should test as non-null.
4346 switch (zcu.getTarget().ptrBitWidth()) {
4347 16 => 0xaaaa,
4348 32 => 0xaaaaaaaa,
4349 64 => 0xaaaaaaaa_aaaaaaaa,
4350 else => unreachable,
4351 };
4352 const llvm_usize = try o.lowerType(pt, Type.usize);
4353 const llvm_ptr_ty = try o.lowerType(pt, ptr_ty);
4354 return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty);
4355 }
4356
4357 /// If the operand type of an atomic operation is not byte sized we need to
4358 /// widen it before using it and then truncate the result.
4359 /// RMW exchange of floating-point values is bitcasted to same-sized integer
4360 /// types to work around a LLVM deficiency when targeting ARM/AArch64.
4361 fn getAtomicAbiType(o: *Object, pt: Zcu.PerThread, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
4362 const zcu = pt.zcu;
4363 const ip = &zcu.intern_pool;
4364 const int_ty = switch (ty.zigTypeTag(zcu)) {
4365 .int => ty,
4366 .@"enum" => ty.intTagType(zcu),
4367 .@"struct" => Type.fromInterned(ip.loadStructType(ty.toIntern()).backingIntTypeUnordered(ip)),
4368 .float => {
4369 if (!is_rmw_xchg) return .none;
4370 return o.builder.intType(@intCast(ty.abiSize(zcu) * 8));
4371 },
4372 .bool => return .i8,
4373 else => return .none,
4374 };
4375 const bit_count = int_ty.intInfo(zcu).bits;
4376 if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
4377 return o.builder.intType(@intCast(int_ty.abiSize(zcu) * 8));
4378 } else {
4379 return .none;
4380 }
4381 }
4382
4383 fn addByValParamAttrs(
4384 o: *Object,
4385 pt: Zcu.PerThread,
4386 attributes: *Builder.FunctionAttributes.Wip,
4387 param_ty: Type,
4388 param_index: u32,
4389 fn_info: InternPool.Key.FuncType,
4390 llvm_arg_i: u32,
4391 ) Allocator.Error!void {
4392 const zcu = pt.zcu;
4393 if (param_ty.isPtrAtRuntime(zcu)) {
4394 const ptr_info = param_ty.ptrInfo(zcu);
4395 if (math.cast(u5, param_index)) |i| {
4396 if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
4397 try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
4398 }
4399 }
4400 if (!param_ty.isPtrLikeOptional(zcu) and
4401 !ptr_info.flags.is_allowzero and
4402 ptr_info.flags.address_space == .generic)
4403 {
4404 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4405 }
4406 switch (fn_info.cc) {
4407 else => {},
4408 .x86_64_interrupt,
4409 .x86_interrupt,
4410 => {
4411 const child_type = try lowerType(o, pt, Type.fromInterned(ptr_info.child));
4412 try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4413 },
4414 }
4415 if (ptr_info.flags.is_const) {
4416 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4417 }
4418 const elem_align = if (ptr_info.flags.alignment != .none)
4419 ptr_info.flags.alignment
4420 else
4421 Type.fromInterned(ptr_info.child).abiAlignment(zcu).max(.@"1");
4422 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align.toLlvm() }, &o.builder);
4423 } else if (ccAbiPromoteInt(fn_info.cc, zcu, param_ty)) |s| switch (s) {
4424 .signed => try attributes.addParamAttr(llvm_arg_i, .signext, &o.builder),
4425 .unsigned => try attributes.addParamAttr(llvm_arg_i, .zeroext, &o.builder),
4426 };
4427 }
4428
4429 fn addByRefParamAttrs(
4430 o: *Object,
4431 attributes: *Builder.FunctionAttributes.Wip,
4432 llvm_arg_i: u32,
4433 alignment: Builder.Alignment,
4434 byval: bool,
4435 param_llvm_ty: Builder.Type,
4436 ) Allocator.Error!void {
4437 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4438 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4439 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = alignment }, &o.builder);
4440 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
4441 }
4442
4443 fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4444 return o.struct_field_map.get(.{
4445 .struct_ty = struct_ty.toIntern(),
4446 .field_index = @intCast(field_index),
4447 });
4448 }
4449
4450 fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {
4451 const name = try o.builder.strtabString(lt_errors_fn_name);
4452 if (o.builder.getGlobal(name)) |llvm_fn| return llvm_fn.ptrConst(&o.builder).kind.function;
4453
4454 const zcu = pt.zcu;
4455 const target = &zcu.root_mod.resolved_target.result;
4456 const function_index = try o.builder.addFunction(
4457 try o.builder.fnType(.i1, &.{try o.errorIntType(pt)}, .normal),
4458 name,
4459 toLlvmAddressSpace(.generic, target),
4460 );
4461
4462 var attributes: Builder.FunctionAttributes.Wip = .{};
4463 defer attributes.deinit(&o.builder);
4464 try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
4465
4466 function_index.setLinkage(.internal, &o.builder);
4467 function_index.setCallConv(.fastcc, &o.builder);
4468 function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4469 return function_index;
4470 }
4471
4472 fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index {
4473 const zcu = pt.zcu;
4474 const ip = &zcu.intern_pool;
4475 const enum_type = ip.loadEnumType(enum_ty.toIntern());
4476
4477 const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern());
4478 if (gop.found_existing) return gop.value_ptr.ptrConst(&o.builder).kind.function;
4479 errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern()));
4480
4481 const usize_ty = try o.lowerType(pt, Type.usize);
4482 const ret_ty = try o.lowerType(pt, Type.slice_const_u8_sentinel_0);
4483 const target = &zcu.root_mod.resolved_target.result;
4484 const function_index = try o.builder.addFunction(
4485 try o.builder.fnType(ret_ty, &.{try o.lowerType(pt, Type.fromInterned(enum_type.tag_ty))}, .normal),
4486 try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_type.name.fmt(ip)}),
4487 toLlvmAddressSpace(.generic, target),
4488 );
4489
4490 var attributes: Builder.FunctionAttributes.Wip = .{};
4491 defer attributes.deinit(&o.builder);
4492 try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
4493
4494 function_index.setLinkage(.internal, &o.builder);
4495 function_index.setCallConv(.fastcc, &o.builder);
4496 function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4497 gop.value_ptr.* = function_index.ptrConst(&o.builder).global;
4498
4499 var wip = try Builder.WipFunction.init(&o.builder, .{
4500 .function = function_index,
4501 .strip = true,
4502 });
4503 defer wip.deinit();
4504 wip.cursor = .{ .block = try wip.block(0, "Entry") };
4505
4506 const bad_value_block = try wip.block(1, "BadValue");
4507 const tag_int_value = wip.arg(0);
4508 var wip_switch =
4509 try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len), .none);
4510 defer wip_switch.finish(&wip);
4511
4512 for (0..enum_type.names.len) |field_index| {
4513 const name = try o.builder.stringNull(enum_type.names.get(ip)[field_index].toSlice(ip));
4514 const name_init = try o.builder.stringConst(name);
4515 const name_variable_index =
4516 try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
4517 try name_variable_index.setInitializer(name_init, &o.builder);
4518 name_variable_index.setLinkage(.private, &o.builder);
4519 name_variable_index.setMutability(.constant, &o.builder);
4520 name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4521 name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
4522
4523 const name_val = try o.builder.structValue(ret_ty, &.{
4524 name_variable_index.toConst(&o.builder),
4525 try o.builder.intConst(usize_ty, name.slice(&o.builder).?.len - 1),
4526 });
4527
4528 const return_block = try wip.block(1, "Name");
4529 const this_tag_int_value = try o.lowerValue(
4530 pt,
4531 (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(),
4532 );
4533 try wip_switch.addCase(this_tag_int_value, return_block, &wip);
4534
4535 wip.cursor = .{ .block = return_block };
4536 _ = try wip.ret(name_val);
4537 }
4538
4539 wip.cursor = .{ .block = bad_value_block };
4540 _ = try wip.@"unreachable"();
4541
4542 try wip.finish();
4543 return function_index;
4544 }
4545};
4546
4547pub const NavGen = struct {
4548 object: *Object,
4549 nav_index: InternPool.Nav.Index,
4550 pt: Zcu.PerThread,
4551 err_msg: ?*Zcu.ErrorMsg,
4552
4553 fn ownerModule(ng: NavGen) *Package.Module {
4554 return ng.pt.zcu.navFileScope(ng.nav_index).mod.?;
4555 }
4556
4557 fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error {
4558 @branchHint(.cold);
4559 assert(ng.err_msg == null);
4560 const o = ng.object;
4561 const gpa = o.gpa;
4562 const src_loc = ng.pt.zcu.navSrcLoc(ng.nav_index);
4563 ng.err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "TODO (LLVM): " ++ format, args);
4564 return error.CodegenFail;
4565 }
4566
4567 fn genDecl(ng: *NavGen) !void {
4568 const o = ng.object;
4569 const pt = ng.pt;
4570 const zcu = pt.zcu;
4571 const ip = &zcu.intern_pool;
4572 const nav_index = ng.nav_index;
4573 const nav = ip.getNav(nav_index);
4574 const resolved = nav.status.fully_resolved;
4575
4576 const lib_name, const linkage, const visibility: Builder.Visibility, const is_threadlocal, const is_dll_import, const is_const, const init_val, const owner_nav = switch (ip.indexToKey(resolved.val)) {
4577 .variable => |variable| .{ .none, .internal, .default, variable.is_threadlocal, false, false, variable.init, variable.owner_nav },
4578 .@"extern" => |@"extern"| .{ @"extern".lib_name, @"extern".linkage, .fromSymbolVisibility(@"extern".visibility), @"extern".is_threadlocal, @"extern".is_dll_import, @"extern".is_const, .none, @"extern".owner_nav },
4579 else => .{ .none, .internal, .default, false, false, true, resolved.val, nav_index },
4580 };
4581 const ty = Type.fromInterned(nav.typeOf(ip));
4582
4583 if (linkage != .internal and ip.isFunctionType(ty.toIntern())) {
4584 _ = try o.resolveLlvmFunction(pt, owner_nav);
4585 } else {
4586 const variable_index = try o.resolveGlobalNav(pt, nav_index);
4587 variable_index.setAlignment(pt.navAlignment(nav_index).toLlvm(), &o.builder);
4588 if (resolved.@"linksection".toSlice(ip)) |section|
4589 variable_index.setSection(try o.builder.string(section), &o.builder);
4590 if (is_const) variable_index.setMutability(.constant, &o.builder);
4591 try variable_index.setInitializer(switch (init_val) {
4592 .none => .no_init,
4593 else => try o.lowerValue(pt, init_val),
4594 }, &o.builder);
4595 variable_index.setVisibility(visibility, &o.builder);
4596
4597 const file_scope = zcu.navFileScopeIndex(nav_index);
4598 const mod = zcu.fileByIndex(file_scope).mod.?;
4599 if (is_threadlocal and !mod.single_threaded)
4600 variable_index.setThreadLocal(.generaldynamic, &o.builder);
4601
4602 const line_number = zcu.navSrcLine(nav_index) + 1;
4603
4604 if (!mod.strip) {
4605 const debug_file = try o.getDebugFile(pt, file_scope);
4606
4607 const debug_global_var = try o.builder.debugGlobalVar(
4608 try o.builder.metadataString(nav.name.toSlice(ip)), // Name
4609 try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name
4610 debug_file, // File
4611 debug_file, // Scope
4612 line_number,
4613 try o.lowerDebugType(pt, ty),
4614 variable_index,
4615 .{ .local = linkage == .internal },
4616 );
4617
4618 const debug_expression = try o.builder.debugExpression(&.{});
4619
4620 const debug_global_var_expression = try o.builder.debugGlobalVarExpression(
4621 debug_global_var,
4622 debug_expression,
4623 );
4624
4625 variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder);
4626 try o.debug_globals.append(o.gpa, debug_global_var_expression);
4627 }
4628 }
4629
4630 switch (linkage) {
4631 .internal => {},
4632 .strong, .weak => {
4633 const global_index = o.nav_map.get(nav_index).?;
4634
4635 const decl_name = decl_name: {
4636 if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") {
4637 if (lib_name.toSlice(ip)) |lib_name_slice| {
4638 if (!std.mem.eql(u8, lib_name_slice, "c")) {
4639 break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice });
4640 }
4641 }
4642 }
4643 break :decl_name try o.builder.strtabString(nav.name.toSlice(ip));
4644 };
4645
4646 if (o.builder.getGlobal(decl_name)) |other_global| {
4647 if (other_global != global_index) {
4648 // Another global already has this name; just use it in place of this global.
4649 try global_index.replace(other_global, &o.builder);
4650 return;
4651 }
4652 }
4653
4654 try global_index.rename(decl_name, &o.builder);
4655 global_index.setUnnamedAddr(.default, &o.builder);
4656 if (is_dll_import) {
4657 global_index.setDllStorageClass(.dllimport, &o.builder);
4658 } else if (zcu.comp.config.dll_export_fns) {
4659 global_index.setDllStorageClass(.default, &o.builder);
4660 }
4661
4662 global_index.setLinkage(switch (linkage) {
4663 .internal => unreachable,
4664 .strong => .external,
4665 .weak => .extern_weak,
4666 .link_once => unreachable,
4667 }, &o.builder);
4668 global_index.setVisibility(visibility, &o.builder);
4669 },
4670 .link_once => unreachable,
4671 }
4672 }
4673};
4674
4675pub const FuncGen = struct {
4676 gpa: Allocator,
4677 ng: *NavGen,
4678 air: Air,
4679 liveness: Air.Liveness,
4680 wip: Builder.WipFunction,
4681 is_naked: bool,
4682 fuzz: ?Fuzz,
4683
4684 file: Builder.Metadata,
4685 scope: Builder.Metadata,
4686
4687 inlined_at: Builder.Metadata.Optional = .none,
4688
4689 base_line: u32,
4690 prev_dbg_line: c_uint,
4691 prev_dbg_column: c_uint,
4692
4693 /// This stores the LLVM values used in a function, such that they can be referred to
4694 /// in other instructions. This table is cleared before every function is generated.
4695 func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
4696
4697 /// If the return type is sret, this is the result pointer. Otherwise null.
4698 /// Note that this can disagree with isByRef for the return type in the case
4699 /// of C ABI functions.
4700 ret_ptr: Builder.Value,
4701 /// Any function that needs to perform Valgrind client requests needs an array alloca
4702 /// instruction, however a maximum of one per function is needed.
4703 valgrind_client_request_array: Builder.Value = .none,
4704 /// These fields are used to refer to the LLVM value of the function parameters
4705 /// in an Arg instruction.
4706 /// This list may be shorter than the list according to the zig type system;
4707 /// it omits 0-bit types. If the function uses sret as the first parameter,
4708 /// this slice does not include it.
4709 args: []const Builder.Value,
4710 arg_index: u32,
4711 arg_inline_index: u32,
4712
4713 err_ret_trace: Builder.Value = .none,
4714
4715 /// This data structure is used to implement breaking to blocks.
4716 blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
4717 parent_bb: Builder.Function.Block.Index,
4718 breaks: *BreakList,
4719 }),
4720
4721 /// Maps `loop` instructions to the bb to branch to to repeat the loop.
4722 loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
4723
4724 /// Maps `loop_switch_br` instructions to the information required to lower
4725 /// dispatches (`switch_dispatch` instructions).
4726 switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
4727
4728 sync_scope: Builder.SyncScope,
4729
4730 disable_intrinsics: bool,
4731
4732 /// Have we seen loads or stores involving `allowzero` pointers?
4733 allowzero_access: bool = false,
4734
4735 pub fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
4736 // LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
4737 // pessimizing optimization for functions with accesses to such pointers.
4738 if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
4739 }
4740
4741 const Fuzz = struct {
4742 counters_variable: Builder.Variable.Index,
4743 pcs: std.ArrayList(Builder.Constant),
4744
4745 fn deinit(f: *Fuzz, gpa: Allocator) void {
4746 f.pcs.deinit(gpa);
4747 f.* = undefined;
4748 }
4749 };
4750
4751 const SwitchDispatchInfo = struct {
4752 /// These are the blocks corresponding to each switch case.
4753 /// The final element corresponds to the `else` case.
4754 /// Slices allocated into `gpa`.
4755 case_blocks: []Builder.Function.Block.Index,
4756 /// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
4757 switch_weights: Builder.Function.Instruction.BrCond.Weights,
4758 /// If not `null`, we have manually constructed a jump table to reach the desired block.
4759 /// `table` can be used if the value is between `min` and `max` inclusive.
4760 /// We perform this lowering manually to avoid some questionable behavior from LLVM.
4761 /// See `airSwitchBr` for details.
4762 jmp_table: ?JmpTable,
4763
4764 const JmpTable = struct {
4765 min: Builder.Constant,
4766 max: Builder.Constant,
4767 in_bounds_hint: enum { none, unpredictable, likely, unlikely },
4768 /// Pointer to the jump table itself, to be used with `indirectbr`.
4769 /// The index into the jump table is the dispatch condition minus `min`.
4770 /// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
4771 table: Builder.Constant,
4772 /// `true` if `table` conatins a reference to the `else` block.
4773 /// In this case, the `indirectbr` must include the `else` block in its target list.
4774 table_includes_else: bool,
4775 };
4776 };
4777
4778 const BreakList = union {
4779 list: std.MultiArrayList(struct {
4780 bb: Builder.Function.Block.Index,
4781 val: Builder.Value,
4782 }),
4783 len: usize,
4784 };
4785
4786 fn deinit(self: *FuncGen) void {
4787 const gpa = self.gpa;
4788 if (self.fuzz) |*f| f.deinit(self.gpa);
4789 self.wip.deinit();
4790 self.func_inst_table.deinit(gpa);
4791 self.blocks.deinit(gpa);
4792 self.loops.deinit(gpa);
4793 var it = self.switch_dispatch_info.valueIterator();
4794 while (it.next()) |info| {
4795 self.gpa.free(info.case_blocks);
4796 }
4797 self.switch_dispatch_info.deinit(gpa);
4798 }
4799
4800 fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error {
4801 @branchHint(.cold);
4802 return self.ng.todo(format, args);
4803 }
4804
4805 fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
4806 const gpa = self.gpa;
4807 const gop = try self.func_inst_table.getOrPut(gpa, inst);
4808 if (gop.found_existing) return gop.value_ptr.*;
4809
4810 const llvm_val = try self.resolveValue((try self.air.value(inst, self.ng.pt)).?);
4811 gop.value_ptr.* = llvm_val.toValue();
4812 return llvm_val.toValue();
4813 }
4814
4815 fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {
4816 const o = self.ng.object;
4817 const pt = self.ng.pt;
4818 const zcu = pt.zcu;
4819 const ty = val.typeOf(zcu);
4820 const llvm_val = try o.lowerValue(pt, val.toIntern());
4821 if (!isByRef(ty, zcu)) return llvm_val;
4822
4823 // We have an LLVM value but we need to create a global constant and
4824 // set the value as its initializer, and then return a pointer to the global.
4825 const target = zcu.getTarget();
4826 const variable_index = try o.builder.addVariable(
4827 .empty,
4828 llvm_val.typeOf(&o.builder),
4829 toLlvmGlobalAddressSpace(.generic, target),
4830 );
4831 try variable_index.setInitializer(llvm_val, &o.builder);
4832 variable_index.setLinkage(.private, &o.builder);
4833 variable_index.setMutability(.constant, &o.builder);
4834 variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4835 variable_index.setAlignment(ty.abiAlignment(zcu).toLlvm(), &o.builder);
4836 return o.builder.convConst(
4837 variable_index.toConst(&o.builder),
4838 try o.builder.ptrType(toLlvmAddressSpace(.generic, target)),
4839 );
4840 }
4841
4842 fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void {
4843 const o = self.ng.object;
4844 const zcu = self.ng.pt.zcu;
4845 const ip = &zcu.intern_pool;
4846 const air_tags = self.air.instructions.items(.tag);
4847 switch (coverage_point) {
4848 .none => {},
4849 .poi => if (self.fuzz) |*fuzz| {
4850 const poi_index = fuzz.pcs.items.len;
4851 const base_ptr = fuzz.counters_variable.toValue(&o.builder);
4852 const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{
4853 try o.builder.intValue(.i32, poi_index),
4854 }, "");
4855 const counter = try self.wip.load(.normal, .i8, ptr, .default, "");
4856 const one = try o.builder.intValue(.i8, 1);
4857 const counter_incremented = try self.wip.bin(.add, counter, one, "");
4858 _ = try self.wip.store(.normal, counter_incremented, ptr, .default);
4859
4860 // LLVM does not allow blockaddress on the entry block.
4861 const pc = if (self.wip.cursor.block == .entry)
4862 self.wip.function.toConst(&o.builder)
4863 else
4864 try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
4865 const gpa = self.gpa;
4866 try fuzz.pcs.append(gpa, pc);
4867 },
4868 }
4869 for (body, 0..) |inst, i| {
4870 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
4871
4872 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4873 // zig fmt: off
4874
4875 // No "scalarize" legalizations are enabled, so these instructions never appear.
4876 .legalize_vec_elem_val => unreachable,
4877 .legalize_vec_store_elem => unreachable,
4878 // No soft float legalizations are enabled.
4879 .legalize_compiler_rt_call => unreachable,
4880
4881 .add => try self.airAdd(inst, .normal),
4882 .add_optimized => try self.airAdd(inst, .fast),
4883 .add_wrap => try self.airAddWrap(inst),
4884 .add_sat => try self.airAddSat(inst),
4885
4886 .sub => try self.airSub(inst, .normal),
4887 .sub_optimized => try self.airSub(inst, .fast),
4888 .sub_wrap => try self.airSubWrap(inst),
4889 .sub_sat => try self.airSubSat(inst),
4890
4891 .mul => try self.airMul(inst, .normal),
4892 .mul_optimized => try self.airMul(inst, .fast),
4893 .mul_wrap => try self.airMulWrap(inst),
4894 .mul_sat => try self.airMulSat(inst),
4895
4896 .add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4897 .sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4898 .mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4899
4900 .div_float => try self.airDivFloat(inst, .normal),
4901 .div_trunc => try self.airDivTrunc(inst, .normal),
4902 .div_floor => try self.airDivFloor(inst, .normal),
4903 .div_exact => try self.airDivExact(inst, .normal),
4904 .rem => try self.airRem(inst, .normal),
4905 .mod => try self.airMod(inst, .normal),
4906 .abs => try self.airAbs(inst),
4907 .ptr_add => try self.airPtrAdd(inst),
4908 .ptr_sub => try self.airPtrSub(inst),
4909 .shl => try self.airShl(inst),
4910 .shl_sat => try self.airShlSat(inst),
4911 .shl_exact => try self.airShlExact(inst),
4912 .min => try self.airMin(inst),
4913 .max => try self.airMax(inst),
4914 .slice => try self.airSlice(inst),
4915 .mul_add => try self.airMulAdd(inst),
4916
4917 .div_float_optimized => try self.airDivFloat(inst, .fast),
4918 .div_trunc_optimized => try self.airDivTrunc(inst, .fast),
4919 .div_floor_optimized => try self.airDivFloor(inst, .fast),
4920 .div_exact_optimized => try self.airDivExact(inst, .fast),
4921 .rem_optimized => try self.airRem(inst, .fast),
4922 .mod_optimized => try self.airMod(inst, .fast),
4923
4924 .add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4925 .sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4926 .mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4927 .shl_with_overflow => try self.airShlWithOverflow(inst),
4928
4929 .bit_and, .bool_and => try self.airAnd(inst),
4930 .bit_or, .bool_or => try self.airOr(inst),
4931 .xor => try self.airXor(inst),
4932 .shr => try self.airShr(inst, false),
4933 .shr_exact => try self.airShr(inst, true),
4934
4935 .sqrt => try self.airUnaryOp(inst, .sqrt),
4936 .sin => try self.airUnaryOp(inst, .sin),
4937 .cos => try self.airUnaryOp(inst, .cos),
4938 .tan => try self.airUnaryOp(inst, .tan),
4939 .exp => try self.airUnaryOp(inst, .exp),
4940 .exp2 => try self.airUnaryOp(inst, .exp2),
4941 .log => try self.airUnaryOp(inst, .log),
4942 .log2 => try self.airUnaryOp(inst, .log2),
4943 .log10 => try self.airUnaryOp(inst, .log10),
4944 .floor => try self.airUnaryOp(inst, .floor),
4945 .ceil => try self.airUnaryOp(inst, .ceil),
4946 .round => try self.airUnaryOp(inst, .round),
4947 .trunc_float => try self.airUnaryOp(inst, .trunc),
4948
4949 .neg => try self.airNeg(inst, .normal),
4950 .neg_optimized => try self.airNeg(inst, .fast),
4951
4952 .cmp_eq => try self.airCmp(inst, .eq, .normal),
4953 .cmp_gt => try self.airCmp(inst, .gt, .normal),
4954 .cmp_gte => try self.airCmp(inst, .gte, .normal),
4955 .cmp_lt => try self.airCmp(inst, .lt, .normal),
4956 .cmp_lte => try self.airCmp(inst, .lte, .normal),
4957 .cmp_neq => try self.airCmp(inst, .neq, .normal),
4958
4959 .cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
4960 .cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
4961 .cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
4962 .cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
4963 .cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
4964 .cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
4965
4966 .cmp_vector => try self.airCmpVector(inst, .normal),
4967 .cmp_vector_optimized => try self.airCmpVector(inst, .fast),
4968 .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
4969
4970 .is_non_null => try self.airIsNonNull(inst, false, .ne),
4971 .is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
4972 .is_null => try self.airIsNonNull(inst, false, .eq),
4973 .is_null_ptr => try self.airIsNonNull(inst, true , .eq),
4974
4975 .is_non_err => try self.airIsErr(inst, .eq, false),
4976 .is_non_err_ptr => try self.airIsErr(inst, .eq, true),
4977 .is_err => try self.airIsErr(inst, .ne, false),
4978 .is_err_ptr => try self.airIsErr(inst, .ne, true),
4979
4980 .alloc => try self.airAlloc(inst),
4981 .ret_ptr => try self.airRetPtr(inst),
4982 .arg => try self.airArg(inst),
4983 .bitcast => try self.airBitCast(inst),
4984 .breakpoint => try self.airBreakpoint(inst),
4985 .ret_addr => try self.airRetAddr(inst),
4986 .frame_addr => try self.airFrameAddress(inst),
4987 .@"try" => try self.airTry(inst, false),
4988 .try_cold => try self.airTry(inst, true),
4989 .try_ptr => try self.airTryPtr(inst, false),
4990 .try_ptr_cold => try self.airTryPtr(inst, true),
4991 .intcast => try self.airIntCast(inst, false),
4992 .intcast_safe => try self.airIntCast(inst, true),
4993 .trunc => try self.airTrunc(inst),
4994 .fptrunc => try self.airFptrunc(inst),
4995 .fpext => try self.airFpext(inst),
4996 .load => try self.airLoad(inst),
4997 .not => try self.airNot(inst),
4998 .store => try self.airStore(inst, false),
4999 .store_safe => try self.airStore(inst, true),
5000 .assembly => try self.airAssembly(inst),
5001 .slice_ptr => try self.airSliceField(inst, 0),
5002 .slice_len => try self.airSliceField(inst, 1),
5003
5004 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
5005 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
5006
5007 .int_from_float => try self.airIntFromFloat(inst, .normal),
5008 .int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
5009 .int_from_float_safe => unreachable, // handled by `legalizeFeatures`
5010 .int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
5011
5012 .array_to_slice => try self.airArrayToSlice(inst),
5013 .float_from_int => try self.airFloatFromInt(inst),
5014 .cmpxchg_weak => try self.airCmpxchg(inst, .weak),
5015 .cmpxchg_strong => try self.airCmpxchg(inst, .strong),
5016 .atomic_rmw => try self.airAtomicRmw(inst),
5017 .atomic_load => try self.airAtomicLoad(inst),
5018 .memset => try self.airMemset(inst, false),
5019 .memset_safe => try self.airMemset(inst, true),
5020 .memcpy => try self.airMemcpy(inst),
5021 .memmove => try self.airMemmove(inst),
5022 .set_union_tag => try self.airSetUnionTag(inst),
5023 .get_union_tag => try self.airGetUnionTag(inst),
5024 .clz => try self.airClzCtz(inst, .ctlz),
5025 .ctz => try self.airClzCtz(inst, .cttz),
5026 .popcount => try self.airBitOp(inst, .ctpop),
5027 .byte_swap => try self.airByteSwap(inst),
5028 .bit_reverse => try self.airBitOp(inst, .bitreverse),
5029 .tag_name => try self.airTagName(inst),
5030 .error_name => try self.airErrorName(inst),
5031 .splat => try self.airSplat(inst),
5032 .select => try self.airSelect(inst),
5033 .shuffle_one => try self.airShuffleOne(inst),
5034 .shuffle_two => try self.airShuffleTwo(inst),
5035 .aggregate_init => try self.airAggregateInit(inst),
5036 .union_init => try self.airUnionInit(inst),
5037 .prefetch => try self.airPrefetch(inst),
5038 .addrspace_cast => try self.airAddrSpaceCast(inst),
5039
5040 .is_named_enum_value => try self.airIsNamedEnumValue(inst),
5041 .error_set_has_value => try self.airErrorSetHasValue(inst),
5042
5043 .reduce => try self.airReduce(inst, .normal),
5044 .reduce_optimized => try self.airReduce(inst, .fast),
5045
5046 .atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
5047 .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
5048 .atomic_store_release => try self.airAtomicStore(inst, .release),
5049 .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
5050
5051 .struct_field_ptr => try self.airStructFieldPtr(inst),
5052 .struct_field_val => try self.airStructFieldVal(inst),
5053
5054 .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
5055 .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
5056 .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
5057 .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
5058
5059 .field_parent_ptr => try self.airFieldParentPtr(inst),
5060
5061 .array_elem_val => try self.airArrayElemVal(inst),
5062 .slice_elem_val => try self.airSliceElemVal(inst),
5063 .slice_elem_ptr => try self.airSliceElemPtr(inst),
5064 .ptr_elem_val => try self.airPtrElemVal(inst),
5065 .ptr_elem_ptr => try self.airPtrElemPtr(inst),
5066
5067 .optional_payload => try self.airOptionalPayload(inst),
5068 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
5069 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
5070
5071 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
5072 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
5073 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
5074 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
5075 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
5076 .err_return_trace => try self.airErrReturnTrace(inst),
5077 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
5078 .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
5079
5080 .wrap_optional => try self.airWrapOptional(body[i..]),
5081 .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
5082 .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
5083
5084 .wasm_memory_size => try self.airWasmMemorySize(inst),
5085 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),
5086
5087 .runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
5088
5089 .inferred_alloc, .inferred_alloc_comptime => unreachable,
5090
5091 .dbg_stmt => try self.airDbgStmt(inst),
5092 .dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
5093 .dbg_var_ptr => try self.airDbgVarPtr(inst),
5094 .dbg_var_val => try self.airDbgVarVal(inst, false),
5095 .dbg_arg_inline => try self.airDbgVarVal(inst, true),
5096
5097 .c_va_arg => try self.airCVaArg(inst),
5098 .c_va_copy => try self.airCVaCopy(inst),
5099 .c_va_end => try self.airCVaEnd(inst),
5100 .c_va_start => try self.airCVaStart(inst),
5101
5102 .work_item_id => try self.airWorkItemId(inst),
5103 .work_group_size => try self.airWorkGroupSize(inst),
5104 .work_group_id => try self.airWorkGroupId(inst),
5105
5106 // Instructions that are known to always be `noreturn` based on their tag.
5107 .br => return self.airBr(inst),
5108 .repeat => return self.airRepeat(inst),
5109 .switch_dispatch => return self.airSwitchDispatch(inst),
5110 .cond_br => return self.airCondBr(inst),
5111 .switch_br => return self.airSwitchBr(inst, false),
5112 .loop_switch_br => return self.airSwitchBr(inst, true),
5113 .loop => return self.airLoop(inst),
5114 .ret => return self.airRet(inst, false),
5115 .ret_safe => return self.airRet(inst, true),
5116 .ret_load => return self.airRetLoad(inst),
5117 .trap => return self.airTrap(inst),
5118 .unreach => return self.airUnreach(inst),
5119
5120 // Instructions which may be `noreturn`.
5121 .block => res: {
5122 const res = try self.airBlock(inst);
5123 if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5124 break :res res;
5125 },
5126 .dbg_inline_block => res: {
5127 const res = try self.airDbgInlineBlock(inst);
5128 if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5129 break :res res;
5130 },
5131 .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: {
5132 const res = try self.airCall(inst, switch (tag) {
5133 .call => .auto,
5134 .call_always_tail => .always_tail,
5135 .call_never_tail => .never_tail,
5136 .call_never_inline => .never_inline,
5137 else => unreachable,
5138 });
5139 // TODO: the AIR we emit for calls is a bit weird - the instruction has
5140 // type `noreturn`, but there are instructions (and maybe a safety check) following
5141 // nonetheless. The `unreachable` or safety check should be emitted by backends instead.
5142 //if (self.typeOfIndex(inst).isNoReturn(mod)) return;
5143 break :res res;
5144 },
5145
5146 // zig fmt: on
5147 };
5148 if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
5149 }
5150 unreachable;
5151 }
5152
5153 fn genBodyDebugScope(
5154 self: *FuncGen,
5155 maybe_inline_func: ?InternPool.Index,
5156 body: []const Air.Inst.Index,
5157 coverage_point: Air.CoveragePoint,
5158 ) Error!void {
5159 if (self.wip.strip) return self.genBody(body, coverage_point);
5160
5161 const old_debug_location = self.wip.debug_location;
5162 const old_file = self.file;
5163 const old_inlined_at = self.inlined_at;
5164 const old_base_line = self.base_line;
5165 defer if (maybe_inline_func) |_| {
5166 self.wip.debug_location = old_debug_location;
5167 self.file = old_file;
5168 self.inlined_at = old_inlined_at;
5169 self.base_line = old_base_line;
5170 };
5171
5172 const old_scope = self.scope;
5173 defer self.scope = old_scope;
5174
5175 if (maybe_inline_func) |inline_func| {
5176 const o = self.ng.object;
5177 const pt = self.ng.pt;
5178 const zcu = pt.zcu;
5179 const ip = &zcu.intern_pool;
5180
5181 const func = zcu.funcInfo(inline_func);
5182 const nav = ip.getNav(func.owner_nav);
5183 const file_scope = zcu.navFileScopeIndex(func.owner_nav);
5184 const mod = zcu.fileByIndex(file_scope).mod.?;
5185
5186 self.file = try o.getDebugFile(pt, file_scope);
5187
5188 self.base_line = zcu.navSrcLine(func.owner_nav);
5189 const line_number = self.base_line + 1;
5190 self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
5191
5192 const fn_ty = try pt.funcType(.{
5193 .param_types = &.{},
5194 .return_type = .void_type,
5195 });
5196
5197 self.scope = try o.builder.debugSubprogram(
5198 self.file,
5199 try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
5200 try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
5201 line_number,
5202 line_number + func.lbrace_line,
5203 try o.lowerDebugType(pt, fn_ty),
5204 .{
5205 .di_flags = .{ .StaticMember = true },
5206 .sp_flags = .{
5207 .Optimized = mod.optimize_mode != .Debug,
5208 .Definition = true,
5209 .LocalToUnit = true, // inline functions cannot be exported
5210 },
5211 },
5212 o.debug_compile_unit.unwrap().?,
5213 );
5214 }
5215
5216 self.scope = try self.ng.object.builder.debugLexicalBlock(
5217 self.scope,
5218 self.file,
5219 self.prev_dbg_line,
5220 self.prev_dbg_column,
5221 );
5222 self.wip.debug_location = .{ .location = .{
5223 .line = self.prev_dbg_line,
5224 .column = self.prev_dbg_column,
5225 .scope = self.scope.toOptional(),
5226 .inlined_at = self.inlined_at,
5227 } };
5228
5229 try self.genBody(body, coverage_point);
5230 }
5231
5232 pub const CallAttr = enum {
5233 Auto,
5234 NeverTail,
5235 NeverInline,
5236 AlwaysTail,
5237 AlwaysInline,
5238 };
5239
5240 fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
5241 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5242 const extra = self.air.extraData(Air.Call, pl_op.payload);
5243 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
5244 const o = self.ng.object;
5245 const pt = self.ng.pt;
5246 const zcu = pt.zcu;
5247 const ip = &zcu.intern_pool;
5248 const callee_ty = self.typeOf(pl_op.operand);
5249 const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
5250 .@"fn" => callee_ty,
5251 .pointer => callee_ty.childType(zcu),
5252 else => unreachable,
5253 };
5254 const fn_info = zcu.typeToFunc(zig_fn_ty).?;
5255 const return_type = Type.fromInterned(fn_info.return_type);
5256 const llvm_fn = try self.resolveInst(pl_op.operand);
5257 const target = zcu.getTarget();
5258 const sret = firstParamSRet(fn_info, zcu, target);
5259
5260 var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
5261 defer llvm_args.deinit();
5262
5263 var attributes: Builder.FunctionAttributes.Wip = .{};
5264 defer attributes.deinit(&o.builder);
5265
5266 if (self.disable_intrinsics) {
5267 try attributes.addFnAttr(.nobuiltin, &o.builder);
5268 }
5269
5270 switch (modifier) {
5271 .auto, .always_tail => {},
5272 .never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
5273 .no_suspend, .always_inline, .compile_time => unreachable,
5274 }
5275
5276 const ret_ptr = if (!sret) null else blk: {
5277 const llvm_ret_ty = try o.lowerType(pt, return_type);
5278 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
5279
5280 const alignment = return_type.abiAlignment(zcu).toLlvm();
5281 const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
5282 try llvm_args.append(ret_ptr);
5283 break :blk ret_ptr;
5284 };
5285
5286 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
5287 if (err_return_tracing) {
5288 assert(self.err_ret_trace != .none);
5289 try llvm_args.append(self.err_ret_trace);
5290 }
5291
5292 var it = iterateParamTypes(o, pt, fn_info);
5293 while (try it.nextCall(self, args)) |lowering| switch (lowering) {
5294 .no_bits => continue,
5295 .byval => {
5296 const arg = args[it.zig_index - 1];
5297 const param_ty = self.typeOf(arg);
5298 const llvm_arg = try self.resolveInst(arg);
5299 const llvm_param_ty = try o.lowerType(pt, param_ty);
5300 if (isByRef(param_ty, zcu)) {
5301 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5302 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
5303 try llvm_args.append(loaded);
5304 } else {
5305 try llvm_args.append(llvm_arg);
5306 }
5307 },
5308 .byref => {
5309 const arg = args[it.zig_index - 1];
5310 const param_ty = self.typeOf(arg);
5311 const llvm_arg = try self.resolveInst(arg);
5312 if (isByRef(param_ty, zcu)) {
5313 try llvm_args.append(llvm_arg);
5314 } else {
5315 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5316 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
5317 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5318 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5319 try llvm_args.append(arg_ptr);
5320 }
5321 },
5322 .byref_mut => {
5323 const arg = args[it.zig_index - 1];
5324 const param_ty = self.typeOf(arg);
5325 const llvm_arg = try self.resolveInst(arg);
5326
5327 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5328 const param_llvm_ty = try o.lowerType(pt, param_ty);
5329 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5330 if (isByRef(param_ty, zcu)) {
5331 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
5332 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
5333 } else {
5334 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5335 }
5336 try llvm_args.append(arg_ptr);
5337 },
5338 .abi_sized_int => {
5339 const arg = args[it.zig_index - 1];
5340 const param_ty = self.typeOf(arg);
5341 const llvm_arg = try self.resolveInst(arg);
5342 const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8));
5343
5344 if (isByRef(param_ty, zcu)) {
5345 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5346 const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, "");
5347 try llvm_args.append(loaded);
5348 } else {
5349 // LLVM does not allow bitcasting structs so we must allocate
5350 // a local, store as one type, and then load as another type.
5351 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5352 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
5353 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
5354 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
5355 try llvm_args.append(loaded);
5356 }
5357 },
5358 .slice => {
5359 const arg = args[it.zig_index - 1];
5360 const llvm_arg = try self.resolveInst(arg);
5361 const ptr = try self.wip.extractValue(llvm_arg, &.{0}, "");
5362 const len = try self.wip.extractValue(llvm_arg, &.{1}, "");
5363 try llvm_args.appendSlice(&.{ ptr, len });
5364 },
5365 .multiple_llvm_types => {
5366 const arg = args[it.zig_index - 1];
5367 const param_ty = self.typeOf(arg);
5368 const llvm_types = it.types_buffer[0..it.types_len];
5369 const llvm_arg = try self.resolveInst(arg);
5370 const is_by_ref = isByRef(param_ty, zcu);
5371 const arg_ptr = if (is_by_ref) llvm_arg else ptr: {
5372 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5373 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5374 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5375 break :ptr ptr;
5376 };
5377
5378 const llvm_ty = try o.builder.structType(.normal, llvm_types);
5379 try llvm_args.ensureUnusedCapacity(it.types_len);
5380 for (llvm_types, 0..) |field_ty, i| {
5381 const alignment =
5382 Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
5383 const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
5384 const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
5385 llvm_args.appendAssumeCapacity(loaded);
5386 }
5387 },
5388 .float_array => |count| {
5389 const arg = args[it.zig_index - 1];
5390 const arg_ty = self.typeOf(arg);
5391 var llvm_arg = try self.resolveInst(arg);
5392 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5393 if (!isByRef(arg_ty, zcu)) {
5394 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5395 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5396 llvm_arg = ptr;
5397 }
5398
5399 const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
5400 const array_ty = try o.builder.arrayType(count, float_ty);
5401
5402 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5403 try llvm_args.append(loaded);
5404 },
5405 .i32_array, .i64_array => |arr_len| {
5406 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
5407 const arg = args[it.zig_index - 1];
5408 const arg_ty = self.typeOf(arg);
5409 var llvm_arg = try self.resolveInst(arg);
5410 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5411 if (!isByRef(arg_ty, zcu)) {
5412 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5413 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5414 llvm_arg = ptr;
5415 }
5416
5417 const array_ty =
5418 try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
5419 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5420 try llvm_args.append(loaded);
5421 },
5422 };
5423
5424 {
5425 // Add argument attributes.
5426 it = iterateParamTypes(o, pt, fn_info);
5427 it.llvm_index += @intFromBool(sret);
5428 it.llvm_index += @intFromBool(err_return_tracing);
5429 while (try it.next()) |lowering| switch (lowering) {
5430 .byval => {
5431 const param_index = it.zig_index - 1;
5432 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5433 if (!isByRef(param_ty, zcu)) {
5434 try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
5435 }
5436 },
5437 .byref => {
5438 const param_index = it.zig_index - 1;
5439 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5440 const param_llvm_ty = try o.lowerType(pt, param_ty);
5441 const alignment = param_ty.abiAlignment(zcu).toLlvm();
5442 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
5443 },
5444 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
5445 // No attributes needed for these.
5446 .no_bits,
5447 .abi_sized_int,
5448 .multiple_llvm_types,
5449 .float_array,
5450 .i32_array,
5451 .i64_array,
5452 => continue,
5453
5454 .slice => {
5455 assert(!it.byval_attr);
5456 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
5457 const ptr_info = param_ty.ptrInfo(zcu);
5458 const llvm_arg_i = it.llvm_index - 2;
5459
5460 if (math.cast(u5, it.zig_index - 1)) |i| {
5461 if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
5462 try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
5463 }
5464 }
5465 if (param_ty.zigTypeTag(zcu) != .optional and
5466 !ptr_info.flags.is_allowzero and
5467 ptr_info.flags.address_space == .generic)
5468 {
5469 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
5470 }
5471 if (ptr_info.flags.is_const) {
5472 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
5473 }
5474 const elem_align = (if (ptr_info.flags.alignment != .none)
5475 @as(InternPool.Alignment, ptr_info.flags.alignment)
5476 else
5477 Type.fromInterned(ptr_info.child).abiAlignment(zcu).max(.@"1")).toLlvm();
5478 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
5479 },
5480 };
5481 }
5482
5483 const call = try self.wip.call(
5484 switch (modifier) {
5485 .auto, .never_inline => .normal,
5486 .never_tail => .notail,
5487 .always_tail => .musttail,
5488 .no_suspend, .always_inline, .compile_time => unreachable,
5489 },
5490 toLlvmCallConvTag(fn_info.cc, target).?,
5491 try attributes.finish(&o.builder),
5492 try o.lowerType(pt, zig_fn_ty),
5493 llvm_fn,
5494 llvm_args.items,
5495 "",
5496 );
5497
5498 if (fn_info.return_type == .noreturn_type and modifier != .always_tail) {
5499 return .none;
5500 }
5501
5502 if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBitsIgnoreComptime(zcu)) {
5503 return .none;
5504 }
5505
5506 const llvm_ret_ty = try o.lowerType(pt, return_type);
5507 if (ret_ptr) |rp| {
5508 if (isByRef(return_type, zcu)) {
5509 return rp;
5510 } else {
5511 // our by-ref status disagrees with sret so we must load.
5512 const return_alignment = return_type.abiAlignment(zcu).toLlvm();
5513 return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
5514 }
5515 }
5516
5517 const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5518
5519 if (abi_ret_ty != llvm_ret_ty) {
5520 // In this case the function return type is honoring the calling convention by having
5521 // a different LLVM type than the usual one. We solve this here at the callsite
5522 // by using our canonical type, then loading it if necessary.
5523 const alignment = return_type.abiAlignment(zcu).toLlvm();
5524 const rp = try self.buildAlloca(abi_ret_ty, alignment);
5525 _ = try self.wip.store(.normal, call, rp, alignment);
5526 return if (isByRef(return_type, zcu))
5527 rp
5528 else
5529 try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
5530 }
5531
5532 if (isByRef(return_type, zcu)) {
5533 // our by-ref status disagrees with sret so we must allocate, store,
5534 // and return the allocation pointer.
5535 const alignment = return_type.abiAlignment(zcu).toLlvm();
5536 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5537 _ = try self.wip.store(.normal, call, rp, alignment);
5538 return rp;
5539 } else {
5540 return call;
5541 }
5542 }
5543
5544 fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void {
5545 const o = fg.ng.object;
5546 const pt = fg.ng.pt;
5547 const zcu = pt.zcu;
5548 const target = zcu.getTarget();
5549 const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin()));
5550 const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
5551 const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav);
5552
5553 const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
5554 if (has_err_trace) assert(fg.err_ret_trace != .none);
5555 _ = try fg.wip.callIntrinsicAssumeCold();
5556 _ = try fg.wip.call(
5557 .normal,
5558 toLlvmCallConvTag(fn_info.cc, target).?,
5559 .none,
5560 panic_global.typeOf(&o.builder),
5561 panic_global.toValue(&o.builder),
5562 if (has_err_trace) &.{fg.err_ret_trace} else &.{},
5563 "",
5564 );
5565 _ = try fg.wip.@"unreachable"();
5566 }
5567
5568 fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void {
5569 const o = self.ng.object;
5570 const pt = self.ng.pt;
5571 const zcu = pt.zcu;
5572 const ip = &zcu.intern_pool;
5573 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5574 const ret_ty = self.typeOf(un_op);
5575
5576 if (self.ret_ptr != .none) {
5577 const ptr_ty = try pt.singleMutPtrType(ret_ty);
5578
5579 const operand = try self.resolveInst(un_op);
5580 const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false;
5581 if (val_is_undef and safety) undef: {
5582 const ptr_info = ptr_ty.ptrInfo(zcu);
5583 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
5584 if (needs_bitmask) {
5585 // TODO: only some bits are to be undef, we cannot write with a simple memset.
5586 // meanwhile, ignore the write rather than stomping over valid bits.
5587 // https://github.com/ziglang/zig/issues/15337
5588 break :undef;
5589 }
5590 const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5591 _ = try self.wip.callMemSet(
5592 self.ret_ptr,
5593 ptr_ty.ptrAlignment(zcu).toLlvm(),
5594 try o.builder.intValue(.i8, 0xaa),
5595 len,
5596 .normal,
5597 self.disable_intrinsics,
5598 );
5599 const owner_mod = self.ng.ownerModule();
5600 if (owner_mod.valgrind) {
5601 try self.valgrindMarkUndef(self.ret_ptr, len);
5602 }
5603 _ = try self.wip.retVoid();
5604 return;
5605 }
5606
5607 const unwrapped_operand = operand.unwrap();
5608 const unwrapped_ret = self.ret_ptr.unwrap();
5609
5610 // Return value was stored previously
5611 if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
5612 _ = try self.wip.retVoid();
5613 return;
5614 }
5615
5616 try self.store(self.ret_ptr, ptr_ty, operand, .none);
5617 _ = try self.wip.retVoid();
5618 return;
5619 }
5620 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).typeOf(ip))).?;
5621 if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
5622 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5623 // Functions with an empty error set are emitted with an error code
5624 // return type and return zero so they can be function pointers coerced
5625 // to functions that return anyerror.
5626 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5627 } else {
5628 _ = try self.wip.retVoid();
5629 }
5630 return;
5631 }
5632
5633 const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5634 const operand = try self.resolveInst(un_op);
5635 const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false;
5636 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5637
5638 if (val_is_undef and safety) {
5639 const llvm_ret_ty = operand.typeOfWip(&self.wip);
5640 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5641 const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5642 _ = try self.wip.callMemSet(
5643 rp,
5644 alignment,
5645 try o.builder.intValue(.i8, 0xaa),
5646 len,
5647 .normal,
5648 self.disable_intrinsics,
5649 );
5650 const owner_mod = self.ng.ownerModule();
5651 if (owner_mod.valgrind) {
5652 try self.valgrindMarkUndef(rp, len);
5653 }
5654 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5655 return;
5656 }
5657
5658 if (isByRef(ret_ty, zcu)) {
5659 // operand is a pointer however self.ret_ptr is null so that means
5660 // we need to return a value.
5661 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
5662 return;
5663 }
5664
5665 const llvm_ret_ty = operand.typeOfWip(&self.wip);
5666 if (abi_ret_ty == llvm_ret_ty) {
5667 _ = try self.wip.ret(operand);
5668 return;
5669 }
5670
5671 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5672 _ = try self.wip.store(.normal, operand, rp, alignment);
5673 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5674 return;
5675 }
5676
5677 fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void {
5678 const o = self.ng.object;
5679 const pt = self.ng.pt;
5680 const zcu = pt.zcu;
5681 const ip = &zcu.intern_pool;
5682 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5683 const ptr_ty = self.typeOf(un_op);
5684 const ret_ty = ptr_ty.childType(zcu);
5685 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).typeOf(ip))).?;
5686 if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
5687 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5688 // Functions with an empty error set are emitted with an error code
5689 // return type and return zero so they can be function pointers coerced
5690 // to functions that return anyerror.
5691 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5692 } else {
5693 _ = try self.wip.retVoid();
5694 }
5695 return;
5696 }
5697 if (self.ret_ptr != .none) {
5698 _ = try self.wip.retVoid();
5699 return;
5700 }
5701 const ptr = try self.resolveInst(un_op);
5702 const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5703 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5704 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
5705 return;
5706 }
5707
5708 fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5709 const o = self.ng.object;
5710 const pt = self.ng.pt;
5711 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5712 const list = try self.resolveInst(ty_op.operand);
5713 const arg_ty = ty_op.ty.toType();
5714 const llvm_arg_ty = try o.lowerType(pt, arg_ty);
5715
5716 return self.wip.vaArg(list, llvm_arg_ty, "");
5717 }
5718
5719 fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5720 const o = self.ng.object;
5721 const pt = self.ng.pt;
5722 const zcu = pt.zcu;
5723 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5724 const src_list = try self.resolveInst(ty_op.operand);
5725 const va_list_ty = ty_op.ty.toType();
5726 const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5727
5728 const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5729 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5730
5731 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
5732 return if (isByRef(va_list_ty, zcu))
5733 dest_list
5734 else
5735 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5736 }
5737
5738 fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5739 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5740 const src_list = try self.resolveInst(un_op);
5741
5742 _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
5743 return .none;
5744 }
5745
5746 fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5747 const o = self.ng.object;
5748 const pt = self.ng.pt;
5749 const zcu = pt.zcu;
5750 const va_list_ty = self.typeOfIndex(inst);
5751 const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5752
5753 const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5754 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5755
5756 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
5757 return if (isByRef(va_list_ty, zcu))
5758 dest_list
5759 else
5760 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5761 }
5762
5763 fn airCmp(
5764 self: *FuncGen,
5765 inst: Air.Inst.Index,
5766 op: math.CompareOperator,
5767 fast: Builder.FastMathKind,
5768 ) !Builder.Value {
5769 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5770 const lhs = try self.resolveInst(bin_op.lhs);
5771 const rhs = try self.resolveInst(bin_op.rhs);
5772 const operand_ty = self.typeOf(bin_op.lhs);
5773
5774 return self.cmp(fast, op, operand_ty, lhs, rhs);
5775 }
5776
5777 fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
5778 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5779 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
5780
5781 const lhs = try self.resolveInst(extra.lhs);
5782 const rhs = try self.resolveInst(extra.rhs);
5783 const vec_ty = self.typeOf(extra.lhs);
5784 const cmp_op = extra.compareOperator();
5785
5786 return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
5787 }
5788
5789 fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5790 const o = self.ng.object;
5791 const pt = self.ng.pt;
5792 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5793 const operand = try self.resolveInst(un_op);
5794 const llvm_fn = try o.getCmpLtErrorsLenFunction(pt);
5795 return self.wip.call(
5796 .normal,
5797 .fastcc,
5798 .none,
5799 llvm_fn.typeOf(&o.builder),
5800 llvm_fn.toValue(&o.builder),
5801 &.{operand},
5802 "",
5803 );
5804 }
5805
5806 fn cmp(
5807 self: *FuncGen,
5808 fast: Builder.FastMathKind,
5809 op: math.CompareOperator,
5810 operand_ty: Type,
5811 lhs: Builder.Value,
5812 rhs: Builder.Value,
5813 ) Allocator.Error!Builder.Value {
5814 const o = self.ng.object;
5815 const pt = self.ng.pt;
5816 const zcu = pt.zcu;
5817 const ip = &zcu.intern_pool;
5818 const scalar_ty = operand_ty.scalarType(zcu);
5819 const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
5820 .@"enum" => scalar_ty.intTagType(zcu),
5821 .int, .bool, .pointer, .error_set => scalar_ty,
5822 .optional => blk: {
5823 const payload_ty = operand_ty.optionalChild(zcu);
5824 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu) or
5825 operand_ty.optionalReprIsPayload(zcu))
5826 {
5827 break :blk operand_ty;
5828 }
5829 // We need to emit instructions to check for equality/inequality
5830 // of optionals that are not pointers.
5831 const is_by_ref = isByRef(scalar_ty, zcu);
5832 const opt_llvm_ty = try o.lowerType(pt, scalar_ty);
5833 const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
5834 const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
5835 const llvm_i2 = try o.builder.intType(2);
5836 const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
5837 const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
5838 const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
5839 const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
5840 const both_null_block = try self.wip.block(1, "BothNull");
5841 const mixed_block = try self.wip.block(1, "Mixed");
5842 const both_pl_block = try self.wip.block(1, "BothNonNull");
5843 const end_block = try self.wip.block(3, "End");
5844 var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
5845 defer wip_switch.finish(&self.wip);
5846 try wip_switch.addCase(
5847 try o.builder.intConst(llvm_i2, 0b00),
5848 both_null_block,
5849 &self.wip,
5850 );
5851 try wip_switch.addCase(
5852 try o.builder.intConst(llvm_i2, 0b11),
5853 both_pl_block,
5854 &self.wip,
5855 );
5856
5857 self.wip.cursor = .{ .block = both_null_block };
5858 _ = try self.wip.br(end_block);
5859
5860 self.wip.cursor = .{ .block = mixed_block };
5861 _ = try self.wip.br(end_block);
5862
5863 self.wip.cursor = .{ .block = both_pl_block };
5864 const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
5865 const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
5866 const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
5867 _ = try self.wip.br(end_block);
5868 const both_pl_block_end = self.wip.cursor.block;
5869
5870 self.wip.cursor = .{ .block = end_block };
5871 const llvm_i1_0 = Builder.Value.false;
5872 const llvm_i1_1 = Builder.Value.true;
5873 const incoming_values: [3]Builder.Value = .{
5874 switch (op) {
5875 .eq => llvm_i1_1,
5876 .neq => llvm_i1_0,
5877 else => unreachable,
5878 },
5879 switch (op) {
5880 .eq => llvm_i1_0,
5881 .neq => llvm_i1_1,
5882 else => unreachable,
5883 },
5884 payload_cmp,
5885 };
5886
5887 const phi = try self.wip.phi(.i1, "");
5888 phi.finish(
5889 &incoming_values,
5890 &.{ both_null_block, mixed_block, both_pl_block_end },
5891 &self.wip,
5892 );
5893 return phi.toValue();
5894 },
5895 .float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
5896 .@"struct" => blk: {
5897 const struct_obj = ip.loadStructType(scalar_ty.toIntern());
5898 assert(struct_obj.layout == .@"packed");
5899 const backing_index = struct_obj.backingIntTypeUnordered(ip);
5900 break :blk Type.fromInterned(backing_index);
5901 },
5902 else => unreachable,
5903 };
5904 const is_signed = int_ty.isSignedInt(zcu);
5905 const cond: Builder.IntegerCondition = switch (op) {
5906 .eq => .eq,
5907 .neq => .ne,
5908 .lt => if (is_signed) .slt else .ult,
5909 .lte => if (is_signed) .sle else .ule,
5910 .gt => if (is_signed) .sgt else .ugt,
5911 .gte => if (is_signed) .sge else .uge,
5912 };
5913 return self.wip.icmp(cond, lhs, rhs, "");
5914 }
5915
5916 fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5917 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5918 const extra = self.air.extraData(Air.Block, ty_pl.payload);
5919 return self.lowerBlock(inst, null, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
5920 }
5921
5922 fn lowerBlock(
5923 self: *FuncGen,
5924 inst: Air.Inst.Index,
5925 maybe_inline_func: ?InternPool.Index,
5926 body: []const Air.Inst.Index,
5927 ) !Builder.Value {
5928 const o = self.ng.object;
5929 const pt = self.ng.pt;
5930 const zcu = pt.zcu;
5931 const inst_ty = self.typeOfIndex(inst);
5932
5933 if (inst_ty.isNoReturn(zcu)) {
5934 try self.genBodyDebugScope(maybe_inline_func, body, .none);
5935 return .none;
5936 }
5937
5938 const have_block_result = inst_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu);
5939
5940 var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
5941 defer if (have_block_result) breaks.list.deinit(self.gpa);
5942
5943 const parent_bb = try self.wip.block(0, "Block");
5944 try self.blocks.putNoClobber(self.gpa, inst, .{
5945 .parent_bb = parent_bb,
5946 .breaks = &breaks,
5947 });
5948 defer assert(self.blocks.remove(inst));
5949
5950 try self.genBodyDebugScope(maybe_inline_func, body, .none);
5951
5952 self.wip.cursor = .{ .block = parent_bb };
5953
5954 // Create a phi node only if the block returns a value.
5955 if (have_block_result) {
5956 const raw_llvm_ty = try o.lowerType(pt, inst_ty);
5957 const llvm_ty: Builder.Type = ty: {
5958 // If the zig tag type is a function, this represents an actual function body; not
5959 // a pointer to it. LLVM IR allows the call instruction to use function bodies instead
5960 // of function pointers, however the phi makes it a runtime value and therefore
5961 // the LLVM type has to be wrapped in a pointer.
5962 if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
5963 break :ty .ptr;
5964 }
5965 break :ty raw_llvm_ty;
5966 };
5967
5968 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
5969 const phi = try self.wip.phi(llvm_ty, "");
5970 phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
5971 return phi.toValue();
5972 } else {
5973 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
5974 return .none;
5975 }
5976 }
5977
5978 fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void {
5979 const zcu = self.ng.pt.zcu;
5980 const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
5981 const block = self.blocks.get(branch.block_inst).?;
5982
5983 // Add the values to the lists only if the break provides a value.
5984 const operand_ty = self.typeOf(branch.operand);
5985 if (operand_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) {
5986 const val = try self.resolveInst(branch.operand);
5987
5988 // For the phi node, we need the basic blocks and the values of the
5989 // break instructions.
5990 try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
5991 } else block.breaks.len += 1;
5992 _ = try self.wip.br(block.parent_bb);
5993 }
5994
5995 fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void {
5996 const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
5997 const loop_bb = self.loops.get(repeat.loop_inst).?;
5998 loop_bb.ptr(&self.wip).incoming += 1;
5999 _ = try self.wip.br(loop_bb);
6000 }
6001
6002 fn lowerSwitchDispatch(
6003 self: *FuncGen,
6004 switch_inst: Air.Inst.Index,
6005 cond_ref: Air.Inst.Ref,
6006 dispatch_info: SwitchDispatchInfo,
6007 ) !void {
6008 const o = self.ng.object;
6009 const pt = self.ng.pt;
6010 const zcu = pt.zcu;
6011 const cond_ty = self.typeOf(cond_ref);
6012 const switch_br = self.air.unwrapSwitch(switch_inst);
6013
6014 if (try self.air.value(cond_ref, pt)) |cond_val| {
6015 // Comptime-known dispatch. Iterate the cases to find the correct
6016 // one, and branch to the corresponding element of `case_blocks`.
6017 var it = switch_br.iterateCases();
6018 const target_case_idx = target: while (it.next()) |case| {
6019 for (case.items) |item| {
6020 const val = Value.fromInterned(item.toInterned().?);
6021 if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
6022 }
6023 for (case.ranges) |range| {
6024 const low = Value.fromInterned(range[0].toInterned().?);
6025 const high = Value.fromInterned(range[1].toInterned().?);
6026 if (cond_val.compareHetero(.gte, low, zcu) and
6027 cond_val.compareHetero(.lte, high, zcu))
6028 {
6029 break :target case.idx;
6030 }
6031 }
6032 } else dispatch_info.case_blocks.len - 1;
6033 const target_block = dispatch_info.case_blocks[target_case_idx];
6034 target_block.ptr(&self.wip).incoming += 1;
6035 _ = try self.wip.br(target_block);
6036 return;
6037 }
6038
6039 // Runtime-known dispatch.
6040 const cond = try self.resolveInst(cond_ref);
6041
6042 if (dispatch_info.jmp_table) |jmp_table| {
6043 // We should use the constructed jump table.
6044 // First, check the bounds to branch to the `else` case if needed.
6045 const inbounds = try self.wip.bin(
6046 .@"and",
6047 try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
6048 try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
6049 "",
6050 );
6051 const jmp_table_block = try self.wip.block(1, "Then");
6052 const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
6053 else_block.ptr(&self.wip).incoming += 1;
6054 _ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
6055 .none => .none,
6056 .unpredictable => .unpredictable,
6057 .likely => .then_likely,
6058 .unlikely => .else_likely,
6059 });
6060
6061 self.wip.cursor = .{ .block = jmp_table_block };
6062
6063 // Figure out the list of blocks we might branch to.
6064 // This includes all case blocks, but it might not include the `else` block if
6065 // the table is dense.
6066 const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
6067 const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
6068
6069 // Make sure to cast the index to a usize so it's not treated as negative!
6070 const table_index = try self.wip.conv(
6071 .unsigned,
6072 try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
6073 try o.lowerType(pt, .usize),
6074 "",
6075 );
6076 const target_ptr_ptr = try self.wip.gep(
6077 .inbounds,
6078 .ptr,
6079 jmp_table.table.toValue(),
6080 &.{table_index},
6081 "",
6082 );
6083 const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
6084
6085 // Do the branch!
6086 _ = try self.wip.indirectbr(target_ptr, target_blocks);
6087
6088 // Mark all target blocks as having one more incoming branch.
6089 for (target_blocks) |case_block| {
6090 case_block.ptr(&self.wip).incoming += 1;
6091 }
6092
6093 return;
6094 }
6095
6096 // We must lower to an actual LLVM `switch` instruction.
6097 // The switch prongs will correspond to our scalar cases. Ranges will
6098 // be handled by conditional branches in the `else` prong.
6099
6100 const llvm_usize = try o.lowerType(pt, Type.usize);
6101 const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
6102 try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
6103 else
6104 cond;
6105
6106 const llvm_cases_len, const last_range_case = info: {
6107 var llvm_cases_len: u32 = 0;
6108 var last_range_case: ?u32 = null;
6109 var it = switch_br.iterateCases();
6110 while (it.next()) |case| {
6111 if (case.ranges.len > 0) last_range_case = case.idx;
6112 llvm_cases_len += @intCast(case.items.len);
6113 }
6114 break :info .{ llvm_cases_len, last_range_case };
6115 };
6116
6117 // The `else` of the LLVM `switch` is the actual `else` prong only
6118 // if there are no ranges. Otherwise, the `else` will have a
6119 // conditional chain before the "true" `else` prong.
6120 const llvm_else_block = if (last_range_case == null)
6121 dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6122 else
6123 try self.wip.block(0, "RangeTest");
6124
6125 llvm_else_block.ptr(&self.wip).incoming += 1;
6126
6127 var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
6128 defer wip_switch.finish(&self.wip);
6129
6130 // Construct the actual cases. Set the cursor to the `else` block so
6131 // we can construct ranges at the same time as scalar cases.
6132 self.wip.cursor = .{ .block = llvm_else_block };
6133
6134 var it = switch_br.iterateCases();
6135 while (it.next()) |case| {
6136 const case_block = dispatch_info.case_blocks[case.idx];
6137
6138 for (case.items) |item| {
6139 const llvm_item = (try self.resolveInst(item)).toConst().?;
6140 const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
6141 try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
6142 else
6143 llvm_item;
6144 try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
6145 }
6146 case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
6147
6148 if (case.ranges.len == 0) continue;
6149
6150 // Add a conditional for the ranges, directing to the relevant bb.
6151 // We don't need to consider `cold` branch hints since that information is stored
6152 // in the target bb body, but we do care about likely/unlikely/unpredictable.
6153
6154 const hint = switch_br.getHint(case.idx);
6155
6156 var range_cond: ?Builder.Value = null;
6157 for (case.ranges) |range| {
6158 const llvm_min = try self.resolveInst(range[0]);
6159 const llvm_max = try self.resolveInst(range[1]);
6160 const cond_part = try self.wip.bin(
6161 .@"and",
6162 try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
6163 try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
6164 "",
6165 );
6166 if (range_cond) |prev| {
6167 range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
6168 } else range_cond = cond_part;
6169 }
6170
6171 // If the check fails, we either branch to the "true" `else` case,
6172 // or to the next range condition.
6173 const range_else_block = if (case.idx == last_range_case.?)
6174 dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6175 else
6176 try self.wip.block(0, "RangeTest");
6177
6178 _ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
6179 .none, .cold => .none,
6180 .unpredictable => .unpredictable,
6181 .likely => .then_likely,
6182 .unlikely => .else_likely,
6183 });
6184 case_block.ptr(&self.wip).incoming += 1;
6185 range_else_block.ptr(&self.wip).incoming += 1;
6186
6187 // Construct the next range conditional (if any) in the false branch.
6188 self.wip.cursor = .{ .block = range_else_block };
6189 }
6190 }
6191
6192 fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void {
6193 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
6194 const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
6195 return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
6196 }
6197
6198 fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
6199 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6200 const cond = try self.resolveInst(pl_op.operand);
6201 const extra = self.air.extraData(Air.CondBr, pl_op.payload);
6202 const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.then_body_len]);
6203 const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
6204
6205 const Hint = enum {
6206 none,
6207 unpredictable,
6208 then_likely,
6209 else_likely,
6210 then_cold,
6211 else_cold,
6212 };
6213 const hint: Hint = switch (extra.data.branch_hints.true) {
6214 .none => switch (extra.data.branch_hints.false) {
6215 .none => .none,
6216 .likely => .else_likely,
6217 .unlikely => .then_likely,
6218 .cold => .else_cold,
6219 .unpredictable => .unpredictable,
6220 },
6221 .likely => switch (extra.data.branch_hints.false) {
6222 .none => .then_likely,
6223 .likely => .unpredictable,
6224 .unlikely => .then_likely,
6225 .cold => .else_cold,
6226 .unpredictable => .unpredictable,
6227 },
6228 .unlikely => switch (extra.data.branch_hints.false) {
6229 .none => .else_likely,
6230 .likely => .else_likely,
6231 .unlikely => .unpredictable,
6232 .cold => .else_cold,
6233 .unpredictable => .unpredictable,
6234 },
6235 .cold => .then_cold,
6236 .unpredictable => .unpredictable,
6237 };
6238
6239 const then_block = try self.wip.block(1, "Then");
6240 const else_block = try self.wip.block(1, "Else");
6241 _ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
6242 .none, .then_cold, .else_cold => .none,
6243 .unpredictable => .unpredictable,
6244 .then_likely => .then_likely,
6245 .else_likely => .else_likely,
6246 });
6247
6248 self.wip.cursor = .{ .block = then_block };
6249 if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
6250 try self.genBodyDebugScope(null, then_body, extra.data.branch_hints.then_cov);
6251
6252 self.wip.cursor = .{ .block = else_block };
6253 if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
6254 try self.genBodyDebugScope(null, else_body, extra.data.branch_hints.else_cov);
6255
6256 // No need to reset the insert cursor since this instruction is noreturn.
6257 }
6258
6259 fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6260 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6261 const err_union = try self.resolveInst(pl_op.operand);
6262 const extra = self.air.extraData(Air.Try, pl_op.payload);
6263 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6264 const err_union_ty = self.typeOf(pl_op.operand);
6265 const is_unused = self.liveness.isUnused(inst);
6266 return lowerTry(self, err_union, body, err_union_ty, false, false, is_unused, err_cold);
6267 }
6268
6269 fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6270 const zcu = self.ng.pt.zcu;
6271 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6272 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
6273 const err_union_ptr = try self.resolveInst(extra.data.ptr);
6274 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]);
6275 const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
6276 const is_unused = self.liveness.isUnused(inst);
6277
6278 self.maybeMarkAllowZeroAccess(self.typeOf(extra.data.ptr).ptrInfo(zcu));
6279
6280 return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold);
6281 }
6282
6283 fn lowerTry(
6284 fg: *FuncGen,
6285 err_union: Builder.Value,
6286 body: []const Air.Inst.Index,
6287 err_union_ty: Type,
6288 operand_is_ptr: bool,
6289 can_elide_load: bool,
6290 is_unused: bool,
6291 err_cold: bool,
6292 ) !Builder.Value {
6293 const o = fg.ng.object;
6294 const pt = fg.ng.pt;
6295 const zcu = pt.zcu;
6296 const payload_ty = err_union_ty.errorUnionPayload(zcu);
6297 const payload_has_bits = payload_ty.hasRuntimeBitsIgnoreComptime(zcu);
6298 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
6299 const error_type = try o.errorIntType(pt);
6300
6301 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6302 const loaded = loaded: {
6303 const access_kind: Builder.MemoryAccessKind =
6304 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
6305
6306 if (!payload_has_bits) {
6307 // TODO add alignment to this load
6308 break :loaded if (operand_is_ptr)
6309 try fg.wip.load(access_kind, error_type, err_union, .default, "")
6310 else
6311 err_union;
6312 }
6313 const err_field_index = try errUnionErrorOffset(payload_ty, pt);
6314 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
6315 const err_field_ptr =
6316 try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");
6317 // TODO add alignment to this load
6318 break :loaded try fg.wip.load(
6319 if (operand_is_ptr) access_kind else .normal,
6320 error_type,
6321 err_field_ptr,
6322 .default,
6323 "",
6324 );
6325 }
6326 break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
6327 };
6328 const zero = try o.builder.intValue(error_type, 0);
6329 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
6330
6331 const return_block = try fg.wip.block(1, "TryRet");
6332 const continue_block = try fg.wip.block(1, "TryCont");
6333 _ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
6334
6335 fg.wip.cursor = .{ .block = return_block };
6336 if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
6337 try fg.genBodyDebugScope(null, body, .poi);
6338
6339 fg.wip.cursor = .{ .block = continue_block };
6340 }
6341 if (is_unused) return .none;
6342 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
6343 const offset = try errUnionPayloadOffset(payload_ty, pt);
6344 if (operand_is_ptr) {
6345 return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6346 } else if (isByRef(err_union_ty, zcu)) {
6347 const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6348 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
6349 if (isByRef(payload_ty, zcu)) {
6350 if (can_elide_load)
6351 return payload_ptr;
6352
6353 return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
6354 }
6355 const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
6356 return fg.wip.load(.normal, load_ty, payload_ptr, payload_alignment, "");
6357 }
6358 return fg.wip.extractValue(err_union, &.{offset}, "");
6359 }
6360
6361 fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
6362 const o = self.ng.object;
6363 const pt = self.ng.pt;
6364 const zcu = pt.zcu;
6365
6366 const switch_br = self.air.unwrapSwitch(inst);
6367
6368 // For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
6369 // For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
6370 // scalar and range cases in the same prong.
6371 // +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
6372 // conditionals to handle ranges.
6373 const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
6374 defer self.gpa.free(case_blocks);
6375 // We set incoming as 0 for now, and increment it as we construct dispatches.
6376 for (case_blocks[0 .. case_blocks.len - 1]) |*b| b.* = try self.wip.block(0, "Case");
6377 case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
6378
6379 // There's a special case here to manually generate a jump table in some cases.
6380 //
6381 // Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
6382 // table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
6383 // use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
6384 // bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
6385 // destroying the cache -- but it also actually generates slightly different jump tables for each case,
6386 // and *a separate conditional branch beforehand* to handle dispatching back to the case we're currently
6387 // within(!!).
6388 //
6389 // This asm is really, really, not what we want. As such, we will construct the jump table manually where
6390 // appropriate (the values are dense and relatively few), and use it when lowering dispatches.
6391
6392 const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
6393 if (!is_dispatch_loop) break :jmp_table null;
6394
6395 // Workaround for:
6396 // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
6397 // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
6398 if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
6399
6400 // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
6401 // about acceptable - it won't fill L1d cache on most CPUs.
6402 const max_table_len = 1024;
6403
6404 const cond_ty = self.typeOf(switch_br.operand);
6405 switch (cond_ty.zigTypeTag(zcu)) {
6406 .bool, .pointer => break :jmp_table null,
6407 .@"enum", .int, .error_set => {},
6408 else => unreachable,
6409 }
6410
6411 if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
6412
6413 // Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
6414 // If they are, then we will construct a jump table.
6415 const min, const max = self.switchCaseItemRange(switch_br);
6416 const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
6417 const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
6418 const table_len = max_int - min_int + 1;
6419 if (table_len > max_table_len) break :jmp_table null;
6420
6421 const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
6422 defer self.gpa.free(table_elems);
6423
6424 // Set them all to the `else` branch, then iterate over the AIR switch
6425 // and replace all values which correspond to other prongs.
6426 @memset(table_elems, try o.builder.blockAddrConst(
6427 self.wip.function,
6428 case_blocks[case_blocks.len - 1],
6429 ));
6430 var item_count: u32 = 0;
6431 var it = switch_br.iterateCases();
6432 while (it.next()) |case| {
6433 const case_block = case_blocks[case.idx];
6434 const case_block_addr = try o.builder.blockAddrConst(
6435 self.wip.function,
6436 case_block,
6437 );
6438 for (case.items) |item| {
6439 const val = Value.fromInterned(item.toInterned().?);
6440 const table_idx = val.toUnsignedInt(zcu) - min_int;
6441 table_elems[@intCast(table_idx)] = case_block_addr;
6442 item_count += 1;
6443 }
6444 for (case.ranges) |range| {
6445 const low = Value.fromInterned(range[0].toInterned().?);
6446 const high = Value.fromInterned(range[1].toInterned().?);
6447 const low_idx = low.toUnsignedInt(zcu) - min_int;
6448 const high_idx = high.toUnsignedInt(zcu) - min_int;
6449 @memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
6450 item_count += @intCast(high_idx + 1 - low_idx);
6451 }
6452 }
6453
6454 const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
6455 const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
6456
6457 const table_variable = try o.builder.addVariable(
6458 try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}),
6459 table_llvm_ty,
6460 .default,
6461 );
6462 try table_variable.setInitializer(table_val, &o.builder);
6463 table_variable.setLinkage(.internal, &o.builder);
6464 table_variable.setUnnamedAddr(.unnamed_addr, &o.builder);
6465
6466 const table_includes_else = item_count != table_len;
6467
6468 break :jmp_table .{
6469 .min = try o.lowerValue(pt, min.toIntern()),
6470 .max = try o.lowerValue(pt, max.toIntern()),
6471 .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
6472 .none, .cold => .none,
6473 .unpredictable => .unpredictable,
6474 .likely => .likely,
6475 .unlikely => .unlikely,
6476 },
6477 .table = table_variable.toConst(&o.builder),
6478 .table_includes_else = table_includes_else,
6479 };
6480 };
6481
6482 const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
6483 if (jmp_table != null) break :weights .none; // not used
6484
6485 // First pass. If any weights are `.unpredictable`, unpredictable.
6486 // If all are `.none` or `.cold`, none.
6487 var any_likely = false;
6488 for (0..switch_br.cases_len) |case_idx| {
6489 switch (switch_br.getHint(@intCast(case_idx))) {
6490 .none, .cold => {},
6491 .likely, .unlikely => any_likely = true,
6492 .unpredictable => break :weights .unpredictable,
6493 }
6494 }
6495 switch (switch_br.getElseHint()) {
6496 .none, .cold => {},
6497 .likely, .unlikely => any_likely = true,
6498 .unpredictable => break :weights .unpredictable,
6499 }
6500 if (!any_likely) break :weights .none;
6501
6502 const llvm_cases_len = llvm_cases_len: {
6503 var len: u32 = 0;
6504 var it = switch_br.iterateCases();
6505 while (it.next()) |case| len += @intCast(case.items.len);
6506 break :llvm_cases_len len;
6507 };
6508
6509 var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
6510 defer self.gpa.free(weights);
6511 var weight_idx: usize = 0;
6512
6513 const branch_weights_str = try o.builder.metadataString("branch_weights");
6514 weights[weight_idx] = branch_weights_str.toMetadata();
6515 weight_idx += 1;
6516
6517 const else_weight: u32 = switch (switch_br.getElseHint()) {
6518 .unpredictable => unreachable,
6519 .none, .cold => 1000,
6520 .likely => 2000,
6521 .unlikely => 1,
6522 };
6523 weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
6524 weight_idx += 1;
6525
6526 var it = switch_br.iterateCases();
6527 while (it.next()) |case| {
6528 const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
6529 .unpredictable => unreachable,
6530 .none, .cold => 1000,
6531 .likely => 2000,
6532 .unlikely => 1,
6533 };
6534 const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
6535 @memset(weights[weight_idx..][0..case.items.len], weight_meta);
6536 weight_idx += case.items.len;
6537 }
6538
6539 assert(weight_idx == weights.len);
6540 break :weights .fromMetadata(try o.builder.metadataTuple(weights));
6541 };
6542
6543 const dispatch_info: SwitchDispatchInfo = .{
6544 .case_blocks = case_blocks,
6545 .switch_weights = weights,
6546 .jmp_table = jmp_table,
6547 };
6548
6549 if (is_dispatch_loop) {
6550 try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
6551 }
6552 defer if (is_dispatch_loop) {
6553 assert(self.switch_dispatch_info.remove(inst));
6554 };
6555
6556 // Generate the initial dispatch.
6557 // If this is a simple `switch_br`, this is the only dispatch.
6558 try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
6559
6560 // Iterate the cases and generate their bodies.
6561 var it = switch_br.iterateCases();
6562 while (it.next()) |case| {
6563 const case_block = case_blocks[case.idx];
6564 self.wip.cursor = .{ .block = case_block };
6565 if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6566 try self.genBodyDebugScope(null, case.body, .none);
6567 }
6568 self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
6569 const else_body = it.elseBody();
6570 if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6571 if (else_body.len > 0) {
6572 try self.genBodyDebugScope(null, it.elseBody(), .none);
6573 } else {
6574 _ = try self.wip.@"unreachable"();
6575 }
6576 }
6577
6578 fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) [2]Value {
6579 const zcu = self.ng.pt.zcu;
6580 var it = switch_br.iterateCases();
6581 var min: ?Value = null;
6582 var max: ?Value = null;
6583 while (it.next()) |case| {
6584 for (case.items) |item| {
6585 const val = Value.fromInterned(item.toInterned().?);
6586 const low = if (min) |m| val.compareHetero(.lt, m, zcu) else true;
6587 const high = if (max) |m| val.compareHetero(.gt, m, zcu) else true;
6588 if (low) min = val;
6589 if (high) max = val;
6590 }
6591 for (case.ranges) |range| {
6592 const vals: [2]Value = .{
6593 Value.fromInterned(range[0].toInterned().?),
6594 Value.fromInterned(range[1].toInterned().?),
6595 };
6596 const low = if (min) |m| vals[0].compareHetero(.lt, m, zcu) else true;
6597 const high = if (max) |m| vals[1].compareHetero(.gt, m, zcu) else true;
6598 if (low) min = vals[0];
6599 if (high) max = vals[1];
6600 }
6601 }
6602 return .{ min.?, max.? };
6603 }
6604
6605 fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
6606 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6607 const loop = self.air.extraData(Air.Block, ty_pl.payload);
6608 const body: []const Air.Inst.Index = @ptrCast(self.air.extra.items[loop.end..][0..loop.data.body_len]);
6609 const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
6610 _ = try self.wip.br(loop_block);
6611
6612 try self.loops.putNoClobber(self.gpa, inst, loop_block);
6613 defer assert(self.loops.remove(inst));
6614
6615 self.wip.cursor = .{ .block = loop_block };
6616 try self.genBodyDebugScope(null, body, .none);
6617 }
6618
6619 fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6620 const o = self.ng.object;
6621 const pt = self.ng.pt;
6622 const zcu = pt.zcu;
6623 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6624 const operand_ty = self.typeOf(ty_op.operand);
6625 const array_ty = operand_ty.childType(zcu);
6626 const llvm_usize = try o.lowerType(pt, Type.usize);
6627 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
6628 const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
6629 const operand = try self.resolveInst(ty_op.operand);
6630 if (!array_ty.hasRuntimeBitsIgnoreComptime(zcu))
6631 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
6632 const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
6633 try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
6634 }, "");
6635 return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
6636 }
6637
6638 fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6639 const o = self.ng.object;
6640 const pt = self.ng.pt;
6641 const zcu = pt.zcu;
6642 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6643
6644 const operand = try self.resolveInst(ty_op.operand);
6645 const operand_ty = self.typeOf(ty_op.operand);
6646 const operand_scalar_ty = operand_ty.scalarType(zcu);
6647 const is_signed_int = operand_scalar_ty.isSignedInt(zcu);
6648
6649 const dest_ty = self.typeOfIndex(inst);
6650 const dest_scalar_ty = dest_ty.scalarType(zcu);
6651 const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6652 const target = zcu.getTarget();
6653
6654 if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
6655 if (is_signed_int) .signed else .unsigned,
6656 operand,
6657 dest_llvm_ty,
6658 "",
6659 );
6660
6661 const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
6662 return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)});
6663 };
6664 const rt_int_ty = try o.builder.intType(rt_int_bits);
6665 var extended = try self.wip.conv(
6666 if (is_signed_int) .signed else .unsigned,
6667 operand,
6668 rt_int_ty,
6669 "",
6670 );
6671 const dest_bits = dest_scalar_ty.floatBits(target);
6672 const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits);
6673 const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits);
6674 const sign_prefix = if (is_signed_int) "" else "un";
6675 const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
6676 sign_prefix,
6677 compiler_rt_operand_abbrev,
6678 compiler_rt_dest_abbrev,
6679 });
6680
6681 var param_type = rt_int_ty;
6682 if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) {
6683 // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6684 // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6685 param_type = try o.builder.vectorType(.normal, 2, .i64);
6686 extended = try self.wip.cast(.bitcast, extended, param_type, "");
6687 }
6688
6689 const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty);
6690 return self.wip.call(
6691 .normal,
6692 .ccc,
6693 .none,
6694 libc_fn.typeOf(&o.builder),
6695 libc_fn.toValue(&o.builder),
6696 &.{extended},
6697 "",
6698 );
6699 }
6700
6701 fn airIntFromFloat(
6702 self: *FuncGen,
6703 inst: Air.Inst.Index,
6704 fast: Builder.FastMathKind,
6705 ) !Builder.Value {
6706 _ = fast;
6707
6708 const o = self.ng.object;
6709 const pt = self.ng.pt;
6710 const zcu = pt.zcu;
6711 const target = zcu.getTarget();
6712 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6713
6714 const operand = try self.resolveInst(ty_op.operand);
6715 const operand_ty = self.typeOf(ty_op.operand);
6716 const operand_scalar_ty = operand_ty.scalarType(zcu);
6717
6718 const dest_ty = self.typeOfIndex(inst);
6719 const dest_scalar_ty = dest_ty.scalarType(zcu);
6720 const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6721
6722 if (intrinsicsAllowed(operand_scalar_ty, target)) {
6723 // TODO set fast math flag
6724 return self.wip.conv(
6725 if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned,
6726 operand,
6727 dest_llvm_ty,
6728 "",
6729 );
6730 }
6731
6732 const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
6733 return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)});
6734 };
6735 const ret_ty = try o.builder.intType(rt_int_bits);
6736 const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
6737 // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6738 // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6739 break :b try o.builder.vectorType(.normal, 2, .i64);
6740 } else ret_ty;
6741
6742 const operand_bits = operand_scalar_ty.floatBits(target);
6743 const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits);
6744
6745 const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits);
6746 const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns";
6747
6748 const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
6749 sign_prefix,
6750 compiler_rt_operand_abbrev,
6751 compiler_rt_dest_abbrev,
6752 });
6753
6754 const operand_llvm_ty = try o.lowerType(pt, operand_ty);
6755 const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
6756 var result = try self.wip.call(
6757 .normal,
6758 .ccc,
6759 .none,
6760 libc_fn.typeOf(&o.builder),
6761 libc_fn.toValue(&o.builder),
6762 &.{operand},
6763 "",
6764 );
6765
6766 if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, "");
6767 if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, "");
6768 return result;
6769 }
6770
6771 fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6772 const zcu = fg.ng.pt.zcu;
6773 return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
6774 }
6775
6776 fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6777 const o = fg.ng.object;
6778 const pt = fg.ng.pt;
6779 const zcu = pt.zcu;
6780 const llvm_usize = try o.lowerType(pt, Type.usize);
6781 switch (ty.ptrSize(zcu)) {
6782 .slice => {
6783 const len = try fg.wip.extractValue(ptr, &.{1}, "");
6784 const elem_ty = ty.childType(zcu);
6785 const abi_size = elem_ty.abiSize(zcu);
6786 if (abi_size == 1) return len;
6787 const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
6788 return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
6789 },
6790 .one => {
6791 const array_ty = ty.childType(zcu);
6792 const elem_ty = array_ty.childType(zcu);
6793 const abi_size = elem_ty.abiSize(zcu);
6794 return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
6795 },
6796 .many, .c => unreachable,
6797 }
6798 }
6799
6800 fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value {
6801 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6802 const operand = try self.resolveInst(ty_op.operand);
6803 return self.wip.extractValue(operand, &.{index}, "");
6804 }
6805
6806 fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {
6807 const o = self.ng.object;
6808 const pt = self.ng.pt;
6809 const zcu = pt.zcu;
6810 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6811 const slice_ptr = try self.resolveInst(ty_op.operand);
6812 const slice_ptr_ty = self.typeOf(ty_op.operand);
6813 const slice_llvm_ty = try o.lowerPtrElemTy(pt, slice_ptr_ty.childType(zcu));
6814
6815 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
6816 }
6817
6818 fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6819 const o = self.ng.object;
6820 const pt = self.ng.pt;
6821 const zcu = pt.zcu;
6822 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6823 const slice_ty = self.typeOf(bin_op.lhs);
6824 const slice = try self.resolveInst(bin_op.lhs);
6825 const index = try self.resolveInst(bin_op.rhs);
6826 const elem_ty = slice_ty.childType(zcu);
6827 const llvm_elem_ty = try o.lowerPtrElemTy(pt, elem_ty);
6828 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6829 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6830 if (isByRef(elem_ty, zcu)) {
6831 self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu));
6832
6833 const slice_align = (slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
6834 return self.loadByRef(ptr, elem_ty, slice_align, if (slice_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
6835 }
6836
6837 self.maybeMarkAllowZeroAccess(slice_ty.ptrInfo(zcu));
6838
6839 return self.load(ptr, slice_ty);
6840 }
6841
6842 fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6843 const o = self.ng.object;
6844 const pt = self.ng.pt;
6845 const zcu = pt.zcu;
6846 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6847 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6848 const slice_ty = self.typeOf(bin_op.lhs);
6849
6850 const slice = try self.resolveInst(bin_op.lhs);
6851 const index = try self.resolveInst(bin_op.rhs);
6852 const llvm_elem_ty = try o.lowerPtrElemTy(pt, slice_ty.childType(zcu));
6853 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6854 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6855 }
6856
6857 fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6858 const o = self.ng.object;
6859 const pt = self.ng.pt;
6860 const zcu = pt.zcu;
6861
6862 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6863 const array_ty = self.typeOf(bin_op.lhs);
6864 const array_llvm_val = try self.resolveInst(bin_op.lhs);
6865 const rhs = try self.resolveInst(bin_op.rhs);
6866 const array_llvm_ty = try o.lowerType(pt, array_ty);
6867 const elem_ty = array_ty.childType(zcu);
6868 if (isByRef(array_ty, zcu)) {
6869 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
6870 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
6871 rhs,
6872 }, "");
6873 if (isByRef(elem_ty, zcu)) {
6874 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6875 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6876 } else {
6877 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
6878 }
6879 }
6880
6881 // This branch can be reached for vectors, which are always by-value.
6882 return self.wip.extractElement(array_llvm_val, rhs, "");
6883 }
6884
6885 fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6886 const o = self.ng.object;
6887 const pt = self.ng.pt;
6888 const zcu = pt.zcu;
6889 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6890 const ptr_ty = self.typeOf(bin_op.lhs);
6891 const elem_ty = ptr_ty.childType(zcu);
6892 const llvm_elem_ty = try o.lowerPtrElemTy(pt, elem_ty);
6893 const base_ptr = try self.resolveInst(bin_op.lhs);
6894 const rhs = try self.resolveInst(bin_op.rhs);
6895 // TODO: when we go fully opaque pointers in LLVM 16 we can remove this branch
6896 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, if (ptr_ty.isSinglePointer(zcu))
6897 // If this is a single-item pointer to an array, we need another index in the GEP.
6898 &.{ try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs }
6899 else
6900 &.{rhs}, "");
6901 if (isByRef(elem_ty, zcu)) {
6902 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6903 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
6904 return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
6905 }
6906
6907 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6908
6909 return self.load(ptr, ptr_ty);
6910 }
6911
6912 fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6913 const o = self.ng.object;
6914 const pt = self.ng.pt;
6915 const zcu = pt.zcu;
6916 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6917 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6918 const ptr_ty = self.typeOf(bin_op.lhs);
6919 const elem_ty = ptr_ty.childType(zcu);
6920 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) return self.resolveInst(bin_op.lhs);
6921
6922 const base_ptr = try self.resolveInst(bin_op.lhs);
6923 const rhs = try self.resolveInst(bin_op.rhs);
6924
6925 const elem_ptr = ty_pl.ty.toType();
6926 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
6927
6928 const llvm_elem_ty = try o.lowerPtrElemTy(pt, elem_ty);
6929 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, if (ptr_ty.isSinglePointer(zcu))
6930 // If this is a single-item pointer to an array, we need another index in the GEP.
6931 &.{ try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs }
6932 else
6933 &.{rhs}, "");
6934 }
6935
6936 fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6937 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6938 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6939 const struct_ptr = try self.resolveInst(struct_field.struct_operand);
6940 const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
6941 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, struct_field.field_index);
6942 }
6943
6944 fn airStructFieldPtrIndex(
6945 self: *FuncGen,
6946 inst: Air.Inst.Index,
6947 field_index: u32,
6948 ) !Builder.Value {
6949 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6950 const struct_ptr = try self.resolveInst(ty_op.operand);
6951 const struct_ptr_ty = self.typeOf(ty_op.operand);
6952 return self.fieldPtr(inst, struct_ptr, struct_ptr_ty, field_index);
6953 }
6954
6955 fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6956 const o = self.ng.object;
6957 const pt = self.ng.pt;
6958 const zcu = pt.zcu;
6959 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6960 const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6961 const struct_ty = self.typeOf(struct_field.struct_operand);
6962 const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
6963 const field_index = struct_field.field_index;
6964 const field_ty = struct_ty.fieldType(field_index, zcu);
6965 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
6966
6967 if (!isByRef(struct_ty, zcu)) {
6968 assert(!isByRef(field_ty, zcu));
6969 switch (struct_ty.zigTypeTag(zcu)) {
6970 .@"struct" => switch (struct_ty.containerLayout(zcu)) {
6971 .@"packed" => {
6972 const struct_type = zcu.typeToStruct(struct_ty).?;
6973 const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);
6974 const containing_int = struct_llvm_val;
6975 const shift_amt =
6976 try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
6977 const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
6978 const elem_llvm_ty = try o.lowerType(pt, field_ty);
6979 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
6980 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6981 const truncated_int =
6982 try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6983 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6984 } else if (field_ty.isPtrAtRuntime(zcu)) {
6985 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6986 const truncated_int =
6987 try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6988 return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
6989 }
6990 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
6991 },
6992 else => {
6993 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6994 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
6995 },
6996 },
6997 .@"union" => {
6998 assert(struct_ty.containerLayout(zcu) == .@"packed");
6999 const containing_int = struct_llvm_val;
7000 const elem_llvm_ty = try o.lowerType(pt, field_ty);
7001 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
7002 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
7003 const truncated_int =
7004 try self.wip.cast(.trunc, containing_int, same_size_int, "");
7005 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
7006 } else if (field_ty.isPtrAtRuntime(zcu)) {
7007 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
7008 const truncated_int =
7009 try self.wip.cast(.trunc, containing_int, same_size_int, "");
7010 return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
7011 }
7012 return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
7013 },
7014 else => unreachable,
7015 }
7016 }
7017
7018 switch (struct_ty.zigTypeTag(zcu)) {
7019 .@"struct" => {
7020 const layout = struct_ty.containerLayout(zcu);
7021 assert(layout != .@"packed");
7022 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
7023 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
7024 const field_ptr =
7025 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
7026 const alignment = struct_ty.fieldAlignment(field_index, zcu);
7027 const field_ptr_ty = try pt.ptrType(.{
7028 .child = field_ty.toIntern(),
7029 .flags = .{ .alignment = alignment },
7030 });
7031 if (isByRef(field_ty, zcu)) {
7032 assert(alignment != .none);
7033 const field_alignment = alignment.toLlvm();
7034 return self.loadByRef(field_ptr, field_ty, field_alignment, .normal);
7035 } else {
7036 return self.load(field_ptr, field_ptr_ty);
7037 }
7038 },
7039 .@"union" => {
7040 const union_llvm_ty = try o.lowerType(pt, struct_ty);
7041 const layout = struct_ty.unionGetLayout(zcu);
7042 const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
7043 const field_ptr =
7044 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
7045 const payload_alignment = layout.payload_align.toLlvm();
7046 if (isByRef(field_ty, zcu)) {
7047 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
7048 } else {
7049 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
7050 }
7051 },
7052 else => unreachable,
7053 }
7054 }
7055
7056 fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7057 const o = self.ng.object;
7058 const pt = self.ng.pt;
7059 const zcu = pt.zcu;
7060 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7061 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7062
7063 const field_ptr = try self.resolveInst(extra.field_ptr);
7064
7065 const parent_ty = ty_pl.ty.toType().childType(zcu);
7066 const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
7067 if (field_offset == 0) return field_ptr;
7068
7069 const res_ty = try o.lowerType(pt, ty_pl.ty.toType());
7070 const llvm_usize = try o.lowerType(pt, Type.usize);
7071
7072 const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
7073 const base_ptr_int = try self.wip.bin(
7074 .@"sub nuw",
7075 field_ptr_int,
7076 try o.builder.intValue(llvm_usize, field_offset),
7077 "",
7078 );
7079 return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
7080 }
7081
7082 fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7083 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7084 const operand = try self.resolveInst(ty_op.operand);
7085
7086 return self.wip.not(operand, "");
7087 }
7088
7089 fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void {
7090 _ = inst;
7091 _ = try self.wip.@"unreachable"();
7092 }
7093
7094 fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7095 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
7096 self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
7097 self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
7098
7099 self.wip.debug_location = .{ .location = .{
7100 .line = self.prev_dbg_line,
7101 .column = self.prev_dbg_column,
7102 .scope = self.scope.toOptional(),
7103 .inlined_at = self.inlined_at,
7104 } };
7105
7106 return .none;
7107 }
7108
7109 fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7110 _ = self;
7111 _ = inst;
7112 return .none;
7113 }
7114
7115 fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7116 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7117 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
7118 self.arg_inline_index = 0;
7119 return self.lowerBlock(inst, extra.data.func, @ptrCast(self.air.extra.items[extra.end..][0..extra.data.body_len]));
7120 }
7121
7122 fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7123 const o = self.ng.object;
7124 const pt = self.ng.pt;
7125 const zcu = pt.zcu;
7126 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7127 const operand = try self.resolveInst(pl_op.operand);
7128 const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7129 const ptr_ty = self.typeOf(pl_op.operand);
7130
7131 const debug_local_var = try o.builder.debugLocalVar(
7132 try o.builder.metadataString(name.toSlice(self.air)),
7133 self.file,
7134 self.scope,
7135 self.prev_dbg_line,
7136 try o.lowerDebugType(pt, ptr_ty.childType(zcu)),
7137 );
7138
7139 _ = try self.wip.callIntrinsic(
7140 .normal,
7141 .none,
7142 .@"dbg.declare",
7143 &.{},
7144 &.{
7145 (try self.wip.debugValue(operand)).toValue(),
7146 debug_local_var.toValue(),
7147 (try o.builder.debugExpression(&.{})).toValue(),
7148 },
7149 "",
7150 );
7151
7152 return .none;
7153 }
7154
7155 fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value {
7156 const o = self.ng.object;
7157 const pt = self.ng.pt;
7158 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7159 const operand = try self.resolveInst(pl_op.operand);
7160 const operand_ty = self.typeOf(pl_op.operand);
7161 const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7162 const name_slice = name.toSlice(self.air);
7163 const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
7164 const debug_local_var = if (is_arg) try o.builder.debugParameter(
7165 metadata_name,
7166 self.file,
7167 self.scope,
7168 self.prev_dbg_line,
7169 try o.lowerDebugType(pt, operand_ty),
7170 arg_no: {
7171 self.arg_inline_index += 1;
7172 break :arg_no self.arg_inline_index;
7173 },
7174 ) else try o.builder.debugLocalVar(
7175 metadata_name,
7176 self.file,
7177 self.scope,
7178 self.prev_dbg_line,
7179 try o.lowerDebugType(pt, operand_ty),
7180 );
7181
7182 const zcu = pt.zcu;
7183 const owner_mod = self.ng.ownerModule();
7184 if (isByRef(operand_ty, zcu)) {
7185 _ = try self.wip.callIntrinsic(
7186 .normal,
7187 .none,
7188 .@"dbg.declare",
7189 &.{},
7190 &.{
7191 (try self.wip.debugValue(operand)).toValue(),
7192 debug_local_var.toValue(),
7193 (try o.builder.debugExpression(&.{})).toValue(),
7194 },
7195 "",
7196 );
7197 } else if (owner_mod.optimize_mode == .Debug and !self.is_naked) {
7198 // We avoid taking this path for naked functions because there's no guarantee that such
7199 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
7200
7201 const alignment = operand_ty.abiAlignment(zcu).toLlvm();
7202 const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
7203 _ = try self.wip.store(.normal, operand, alloca, alignment);
7204 _ = try self.wip.callIntrinsic(
7205 .normal,
7206 .none,
7207 .@"dbg.declare",
7208 &.{},
7209 &.{
7210 (try self.wip.debugValue(alloca)).toValue(),
7211 debug_local_var.toValue(),
7212 (try o.builder.debugExpression(&.{})).toValue(),
7213 },
7214 "",
7215 );
7216 } else {
7217 _ = try self.wip.callIntrinsic(
7218 .normal,
7219 .none,
7220 .@"dbg.value",
7221 &.{},
7222 &.{
7223 (try self.wip.debugValue(operand)).toValue(),
7224 debug_local_var.toValue(),
7225 (try o.builder.debugExpression(&.{})).toValue(),
7226 },
7227 "",
7228 );
7229 }
7230 return .none;
7231 }
7232
7233 fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7234 // Eventually, the Zig compiler needs to be reworked to have inline
7235 // assembly go through the same parsing code regardless of backend, and
7236 // have LLVM-flavored inline assembly be *output* from that assembler.
7237 // We don't have such an assembler implemented yet though. For now,
7238 // this implementation feeds the inline assembly code directly to LLVM.
7239
7240 const o = self.ng.object;
7241 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7242 const extra = self.air.extraData(Air.Asm, ty_pl.payload);
7243 const is_volatile = extra.data.flags.is_volatile;
7244 const outputs_len = extra.data.flags.outputs_len;
7245 const gpa = self.gpa;
7246 var extra_i: usize = extra.end;
7247
7248 const outputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..outputs_len]);
7249 extra_i += outputs.len;
7250 const inputs: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra_i..][0..extra.data.inputs_len]);
7251 extra_i += inputs.len;
7252
7253 var llvm_constraints: std.ArrayList(u8) = .empty;
7254 defer llvm_constraints.deinit(gpa);
7255
7256 var arena_allocator = std.heap.ArenaAllocator.init(gpa);
7257 defer arena_allocator.deinit();
7258 const arena = arena_allocator.allocator();
7259
7260 // The exact number of return / parameter values depends on which output values
7261 // are passed by reference as indirect outputs (determined below).
7262 const max_return_count = outputs.len;
7263 const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
7264 const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
7265 const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
7266
7267 const max_param_count = max_return_count + inputs.len + outputs.len;
7268 const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
7269 const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
7270 // This stores whether we need to add an elementtype attribute and
7271 // if so, the element type itself.
7272 const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
7273 const pt = self.ng.pt;
7274 const zcu = pt.zcu;
7275 const target = zcu.getTarget();
7276
7277 var llvm_ret_i: usize = 0;
7278 var llvm_param_i: usize = 0;
7279 var total_i: usize = 0;
7280
7281 var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
7282 try name_map.ensureUnusedCapacity(arena, max_param_count);
7283
7284 var rw_extra_i = extra_i;
7285 for (outputs, llvm_ret_indirect, llvm_rw_vals) |output, *is_indirect, *llvm_rw_val| {
7286 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7287 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[extra_i..]), 0);
7288 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7289 // This equation accounts for the fact that even if we have exactly 4 bytes
7290 // for the string, we still use the next u32 for the null terminator.
7291 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
7292
7293 try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
7294 if (total_i != 0) {
7295 llvm_constraints.appendAssumeCapacity(',');
7296 }
7297 llvm_constraints.appendAssumeCapacity('=');
7298
7299 if (output != .none) {
7300 const output_inst = try self.resolveInst(output);
7301 const output_ty = self.typeOf(output);
7302 assert(output_ty.zigTypeTag(zcu) == .pointer);
7303 const elem_llvm_ty = try o.lowerPtrElemTy(pt, output_ty.childType(zcu));
7304
7305 switch (constraint[0]) {
7306 '=' => {},
7307 '+' => llvm_rw_val.* = output_inst,
7308 else => return self.todo("unsupported output constraint on output type '{c}'", .{
7309 constraint[0],
7310 }),
7311 }
7312
7313 self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
7314
7315 // Pass any non-return outputs indirectly, if the constraint accepts a memory location
7316 is_indirect.* = constraintAllowsMemory(constraint);
7317 if (is_indirect.*) {
7318 // Pass the result by reference as an indirect output (e.g. "=*m")
7319 llvm_constraints.appendAssumeCapacity('*');
7320
7321 llvm_param_values[llvm_param_i] = output_inst;
7322 llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
7323 llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
7324 llvm_param_i += 1;
7325 } else {
7326 // Pass the result directly (e.g. "=r")
7327 llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
7328 llvm_ret_i += 1;
7329 }
7330 } else {
7331 switch (constraint[0]) {
7332 '=' => {},
7333 else => return self.todo("unsupported output constraint on result type '{s}'", .{
7334 constraint,
7335 }),
7336 }
7337
7338 is_indirect.* = false;
7339
7340 const ret_ty = self.typeOfIndex(inst);
7341 llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty);
7342 llvm_ret_i += 1;
7343 }
7344
7345 // LLVM uses commas internally to separate different constraints,
7346 // alternative constraints are achieved with pipes.
7347 // We still allow the user to use commas in a way that is similar
7348 // to GCC's inline assembly.
7349 // http://llvm.org/docs/LangRef.html#constraint-codes
7350 for (constraint[1..]) |byte| {
7351 switch (byte) {
7352 ',' => llvm_constraints.appendAssumeCapacity('|'),
7353 '*' => {}, // Indirect outputs are handled above
7354 else => llvm_constraints.appendAssumeCapacity(byte),
7355 }
7356 }
7357
7358 if (!std.mem.eql(u8, name, "_")) {
7359 const gop = name_map.getOrPutAssumeCapacity(name);
7360 if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
7361 gop.value_ptr.* = @intCast(total_i);
7362 }
7363 total_i += 1;
7364 }
7365
7366 for (inputs) |input| {
7367 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[extra_i..]);
7368 const constraint = std.mem.sliceTo(extra_bytes, 0);
7369 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7370 // This equation accounts for the fact that even if we have exactly 4 bytes
7371 // for the string, we still use the next u32 for the null terminator.
7372 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
7373
7374 const arg_llvm_value = try self.resolveInst(input);
7375 const arg_ty = self.typeOf(input);
7376 const is_by_ref = isByRef(arg_ty, zcu);
7377 if (is_by_ref) {
7378 if (constraintAllowsMemory(constraint)) {
7379 llvm_param_values[llvm_param_i] = arg_llvm_value;
7380 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7381 } else {
7382 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7383 const arg_llvm_ty = try o.lowerType(pt, arg_ty);
7384 const load_inst =
7385 try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
7386 llvm_param_values[llvm_param_i] = load_inst;
7387 llvm_param_types[llvm_param_i] = arg_llvm_ty;
7388 }
7389 } else {
7390 if (constraintAllowsRegister(constraint)) {
7391 llvm_param_values[llvm_param_i] = arg_llvm_value;
7392 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7393 } else {
7394 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7395 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
7396 _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
7397 llvm_param_values[llvm_param_i] = arg_ptr;
7398 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
7399 }
7400 }
7401
7402 try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
7403 if (total_i != 0) {
7404 llvm_constraints.appendAssumeCapacity(',');
7405 }
7406 for (constraint) |byte| {
7407 llvm_constraints.appendAssumeCapacity(switch (byte) {
7408 ',' => '|',
7409 else => byte,
7410 });
7411 }
7412
7413 if (!std.mem.eql(u8, name, "_")) {
7414 const gop = name_map.getOrPutAssumeCapacity(name);
7415 if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
7416 gop.value_ptr.* = @intCast(total_i);
7417 }
7418
7419 // In the case of indirect inputs, LLVM requires the callsite to have
7420 // an elementtype(<ty>) attribute.
7421 llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
7422 if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
7423
7424 break :blk try o.lowerPtrElemTy(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu));
7425 } else .none;
7426
7427 llvm_param_i += 1;
7428 total_i += 1;
7429 }
7430
7431 for (outputs, llvm_ret_indirect, llvm_rw_vals, 0..) |output, is_indirect, llvm_rw_val, output_index| {
7432 const extra_bytes = std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]);
7433 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra.items[rw_extra_i..]), 0);
7434 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
7435 // This equation accounts for the fact that even if we have exactly 4 bytes
7436 // for the string, we still use the next u32 for the null terminator.
7437 rw_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
7438
7439 if (constraint[0] != '+') continue;
7440
7441 const rw_ty = self.typeOf(output);
7442 const llvm_elem_ty = try o.lowerPtrElemTy(pt, rw_ty.childType(zcu));
7443 if (is_indirect) {
7444 llvm_param_values[llvm_param_i] = llvm_rw_val;
7445 llvm_param_types[llvm_param_i] = llvm_rw_val.typeOfWip(&self.wip);
7446 } else {
7447 const alignment = rw_ty.abiAlignment(zcu).toLlvm();
7448 const loaded = try self.wip.load(
7449 if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7450 llvm_elem_ty,
7451 llvm_rw_val,
7452 alignment,
7453 "",
7454 );
7455 llvm_param_values[llvm_param_i] = loaded;
7456 llvm_param_types[llvm_param_i] = llvm_elem_ty;
7457 }
7458
7459 try llvm_constraints.print(gpa, ",{d}", .{output_index});
7460
7461 // In the case of indirect inputs, LLVM requires the callsite to have
7462 // an elementtype(<ty>) attribute.
7463 llvm_param_attrs[llvm_param_i] = if (is_indirect) llvm_elem_ty else .none;
7464
7465 llvm_param_i += 1;
7466 total_i += 1;
7467 }
7468
7469 const ip = &zcu.intern_pool;
7470 const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
7471 const struct_type: Type = .fromInterned(aggregate.ty);
7472 if (total_i != 0) try llvm_constraints.append(gpa, ',');
7473 switch (aggregate.storage) {
7474 .elems => |elems| for (elems, 0..) |elem, i| {
7475 switch (elem) {
7476 .bool_true => {
7477 const name = struct_type.structFieldName(i, zcu).toSlice(ip).?;
7478 total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
7479 },
7480 .bool_false => continue,
7481 else => unreachable,
7482 }
7483 },
7484 .repeated_elem => |elem| switch (elem) {
7485 .bool_true => for (0..struct_type.structFieldCount(zcu)) |i| {
7486 const name = struct_type.structFieldName(i, zcu).toSlice(ip).?;
7487 total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
7488 },
7489 .bool_false => {},
7490 else => unreachable,
7491 },
7492 .bytes => @panic("TODO"),
7493 }
7494
7495 // We have finished scanning through all inputs/outputs, so the number of
7496 // parameters and return values is known.
7497 const param_count = llvm_param_i;
7498 const return_count = llvm_ret_i;
7499
7500 // For some targets, Clang unconditionally adds some clobbers to all inline assembly.
7501 // While this is probably not strictly necessary, if we don't follow Clang's lead
7502 // here then we may risk tripping LLVM bugs since anything not used by Clang tends
7503 // to be buggy and regress often.
7504 switch (target.cpu.arch) {
7505 .x86_64, .x86 => {
7506 try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
7507 total_i += 3;
7508 },
7509 .mips, .mipsel, .mips64, .mips64el => {
7510 try llvm_constraints.appendSlice(gpa, "~{$1},");
7511 total_i += 1;
7512 },
7513 else => {},
7514 }
7515
7516 if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
7517
7518 const asm_source = std.mem.sliceAsBytes(self.air.extra.items[extra_i..])[0..extra.data.source_len];
7519
7520 // hackety hacks until stage2 has proper inline asm in the frontend.
7521 var rendered_template = std.array_list.Managed(u8).init(gpa);
7522 defer rendered_template.deinit();
7523
7524 const State = enum { start, percent, input, modifier };
7525
7526 var state: State = .start;
7527
7528 var name_start: usize = undefined;
7529 var modifier_start: usize = undefined;
7530 for (asm_source, 0..) |byte, i| {
7531 switch (state) {
7532 .start => switch (byte) {
7533 '%' => state = .percent,
7534 '$' => try rendered_template.appendSlice("$$"),
7535 else => try rendered_template.append(byte),
7536 },
7537 .percent => switch (byte) {
7538 '%' => {
7539 try rendered_template.append('%');
7540 state = .start;
7541 },
7542 '[' => {
7543 try rendered_template.append('$');
7544 try rendered_template.append('{');
7545 name_start = i + 1;
7546 state = .input;
7547 },
7548 '=' => {
7549 try rendered_template.appendSlice("${:uid}");
7550 state = .start;
7551 },
7552 else => {
7553 try rendered_template.append('%');
7554 try rendered_template.append(byte);
7555 state = .start;
7556 },
7557 },
7558 .input => switch (byte) {
7559 ']', ':' => {
7560 const name = asm_source[name_start..i];
7561
7562 const index = name_map.get(name) orelse {
7563 // we should validate the assembly in Sema; by now it is too late
7564 return self.todo("unknown input or output name: '{s}'", .{name});
7565 };
7566 try rendered_template.print("{d}", .{index});
7567 if (byte == ':') {
7568 try rendered_template.append(':');
7569 modifier_start = i + 1;
7570 state = .modifier;
7571 } else {
7572 try rendered_template.append('}');
7573 state = .start;
7574 }
7575 },
7576 else => {},
7577 },
7578 .modifier => switch (byte) {
7579 ']' => {
7580 try rendered_template.appendSlice(asm_source[modifier_start..i]);
7581 try rendered_template.append('}');
7582 state = .start;
7583 },
7584 else => {},
7585 },
7586 }
7587 }
7588
7589 var attributes: Builder.FunctionAttributes.Wip = .{};
7590 defer attributes.deinit(&o.builder);
7591 for (llvm_param_attrs[0..param_count], 0..) |llvm_elem_ty, i| if (llvm_elem_ty != .none)
7592 try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
7593
7594 const ret_llvm_ty = switch (return_count) {
7595 0 => .void,
7596 1 => llvm_ret_types[0],
7597 else => try o.builder.structType(.normal, llvm_ret_types),
7598 };
7599 const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
7600 const call = try self.wip.callAsm(
7601 try attributes.finish(&o.builder),
7602 llvm_fn_ty,
7603 .{ .sideeffect = is_volatile },
7604 try o.builder.string(rendered_template.items),
7605 try o.builder.string(llvm_constraints.items),
7606 llvm_param_values[0..param_count],
7607 "",
7608 );
7609
7610 var ret_val = call;
7611 llvm_ret_i = 0;
7612 for (outputs, 0..) |output, i| {
7613 if (llvm_ret_indirect[i]) continue;
7614
7615 const output_value = if (return_count > 1)
7616 try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
7617 else
7618 call;
7619
7620 if (output != .none) {
7621 const output_ptr = try self.resolveInst(output);
7622 const output_ptr_ty = self.typeOf(output);
7623 const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
7624 _ = try self.wip.store(
7625 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7626 output_value,
7627 output_ptr,
7628 alignment,
7629 );
7630 } else {
7631 ret_val = output_value;
7632 }
7633 llvm_ret_i += 1;
7634 }
7635
7636 return ret_val;
7637 }
7638
7639 fn airIsNonNull(
7640 self: *FuncGen,
7641 inst: Air.Inst.Index,
7642 operand_is_ptr: bool,
7643 cond: Builder.IntegerCondition,
7644 ) !Builder.Value {
7645 const o = self.ng.object;
7646 const pt = self.ng.pt;
7647 const zcu = pt.zcu;
7648 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7649 const operand = try self.resolveInst(un_op);
7650 const operand_ty = self.typeOf(un_op);
7651 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7652 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7653 const payload_ty = optional_ty.optionalChild(zcu);
7654
7655 const access_kind: Builder.MemoryAccessKind =
7656 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7657
7658 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7659
7660 if (optional_ty.optionalReprIsPayload(zcu)) {
7661 const loaded = if (operand_is_ptr)
7662 try self.wip.load(access_kind, optional_llvm_ty, operand, .default, "")
7663 else
7664 operand;
7665 if (payload_ty.isSlice(zcu)) {
7666 const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
7667 const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(
7668 payload_ty.ptrAddressSpace(zcu),
7669 zcu.getTarget(),
7670 ));
7671 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
7672 }
7673 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
7674 }
7675
7676 comptime assert(optional_layout_version == 3);
7677
7678 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7679 const loaded = if (operand_is_ptr)
7680 try self.wip.load(access_kind, optional_llvm_ty, operand, .default, "")
7681 else
7682 operand;
7683 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
7684 }
7685
7686 const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);
7687 return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
7688 }
7689
7690 fn airIsErr(
7691 self: *FuncGen,
7692 inst: Air.Inst.Index,
7693 cond: Builder.IntegerCondition,
7694 operand_is_ptr: bool,
7695 ) !Builder.Value {
7696 const o = self.ng.object;
7697 const pt = self.ng.pt;
7698 const zcu = pt.zcu;
7699 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7700 const operand = try self.resolveInst(un_op);
7701 const operand_ty = self.typeOf(un_op);
7702 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7703 const payload_ty = err_union_ty.errorUnionPayload(zcu);
7704 const error_type = try o.errorIntType(pt);
7705 const zero = try o.builder.intValue(error_type, 0);
7706
7707 const access_kind: Builder.MemoryAccessKind =
7708 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7709
7710 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7711 const val: Builder.Constant = switch (cond) {
7712 .eq => .true, // 0 == 0
7713 .ne => .false, // 0 != 0
7714 else => unreachable,
7715 };
7716 return val.toValue();
7717 }
7718
7719 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7720
7721 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7722 const loaded = if (operand_is_ptr)
7723 try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, .default, "")
7724 else
7725 operand;
7726 return self.wip.icmp(cond, loaded, zero, "");
7727 }
7728
7729 const err_field_index = try errUnionErrorOffset(payload_ty, pt);
7730
7731 const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {
7732 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7733 const err_field_ptr =
7734 try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
7735 break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
7736 } else try self.wip.extractValue(operand, &.{err_field_index}, "");
7737 return self.wip.icmp(cond, loaded, zero, "");
7738 }
7739
7740 fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7741 const o = self.ng.object;
7742 const pt = self.ng.pt;
7743 const zcu = pt.zcu;
7744 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7745 const operand = try self.resolveInst(ty_op.operand);
7746 const optional_ty = self.typeOf(ty_op.operand).childType(zcu);
7747 const payload_ty = optional_ty.optionalChild(zcu);
7748 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7749 // We have a pointer to a zero-bit value and we need to return
7750 // a pointer to a zero-bit value.
7751 return operand;
7752 }
7753 if (optional_ty.optionalReprIsPayload(zcu)) {
7754 // The payload and the optional are the same value.
7755 return operand;
7756 }
7757 return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
7758 }
7759
7760 fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7761 comptime assert(optional_layout_version == 3);
7762
7763 const o = self.ng.object;
7764 const pt = self.ng.pt;
7765 const zcu = pt.zcu;
7766 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7767 const operand = try self.resolveInst(ty_op.operand);
7768 const optional_ptr_ty = self.typeOf(ty_op.operand);
7769 const optional_ty = optional_ptr_ty.childType(zcu);
7770 const payload_ty = optional_ty.optionalChild(zcu);
7771 const non_null_bit = try o.builder.intValue(.i8, 1);
7772
7773 const access_kind: Builder.MemoryAccessKind =
7774 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7775
7776 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7777 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7778
7779 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
7780 _ = try self.wip.store(access_kind, non_null_bit, operand, .default);
7781 return operand;
7782 }
7783 if (optional_ty.optionalReprIsPayload(zcu)) {
7784 // The payload and the optional are the same value.
7785 // Setting to non-null will be done when the payload is set.
7786 return operand;
7787 }
7788
7789 // First set the non-null bit.
7790 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7791 const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");
7792
7793 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7794
7795 // TODO set alignment on this store
7796 _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
7797
7798 // Then return the payload pointer (only if it's used).
7799 if (self.liveness.isUnused(inst)) return .none;
7800
7801 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
7802 }
7803
7804 fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7805 const o = self.ng.object;
7806 const pt = self.ng.pt;
7807 const zcu = pt.zcu;
7808 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7809 const operand = try self.resolveInst(ty_op.operand);
7810 const optional_ty = self.typeOf(ty_op.operand);
7811 const payload_ty = self.typeOfIndex(inst);
7812 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
7813
7814 if (optional_ty.optionalReprIsPayload(zcu)) {
7815 // Payload value is the same as the optional value.
7816 return operand;
7817 }
7818
7819 const opt_llvm_ty = try o.lowerType(pt, optional_ty);
7820 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
7821 }
7822
7823 fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
7824 const o = self.ng.object;
7825 const pt = self.ng.pt;
7826 const zcu = pt.zcu;
7827 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7828 const operand = try self.resolveInst(ty_op.operand);
7829 const operand_ty = self.typeOf(ty_op.operand);
7830 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7831 const result_ty = self.typeOfIndex(inst);
7832 const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
7833
7834 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7835 return if (operand_is_ptr) operand else .none;
7836 }
7837 const offset = try errUnionPayloadOffset(payload_ty, pt);
7838 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7839 if (operand_is_ptr) {
7840 return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7841 } else if (isByRef(err_union_ty, zcu)) {
7842 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
7843 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7844 if (isByRef(payload_ty, zcu)) {
7845 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
7846 }
7847 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
7848 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
7849 }
7850 return self.wip.extractValue(operand, &.{offset}, "");
7851 }
7852
7853 fn airErrUnionErr(
7854 self: *FuncGen,
7855 inst: Air.Inst.Index,
7856 operand_is_ptr: bool,
7857 ) !Builder.Value {
7858 const o = self.ng.object;
7859 const pt = self.ng.pt;
7860 const zcu = pt.zcu;
7861 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7862 const operand = try self.resolveInst(ty_op.operand);
7863 const operand_ty = self.typeOf(ty_op.operand);
7864 const error_type = try o.errorIntType(pt);
7865 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7866 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7867 if (operand_is_ptr) {
7868 return operand;
7869 } else {
7870 return o.builder.intValue(error_type, 0);
7871 }
7872 }
7873
7874 const access_kind: Builder.MemoryAccessKind =
7875 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7876
7877 const payload_ty = err_union_ty.errorUnionPayload(zcu);
7878 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7879 if (!operand_is_ptr) return operand;
7880
7881 self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7882
7883 return self.wip.load(access_kind, error_type, operand, .default, "");
7884 }
7885
7886 const offset = try errUnionErrorOffset(payload_ty, pt);
7887
7888 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
7889 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7890
7891 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7892 const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7893 return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
7894 }
7895
7896 return self.wip.extractValue(operand, &.{offset}, "");
7897 }
7898
7899 fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7900 const o = self.ng.object;
7901 const pt = self.ng.pt;
7902 const zcu = pt.zcu;
7903 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7904 const operand = try self.resolveInst(ty_op.operand);
7905 const err_union_ptr_ty = self.typeOf(ty_op.operand);
7906 const err_union_ty = err_union_ptr_ty.childType(zcu);
7907
7908 const payload_ty = err_union_ty.errorUnionPayload(zcu);
7909 const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0);
7910
7911 const access_kind: Builder.MemoryAccessKind =
7912 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7913
7914 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
7915 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7916
7917 _ = try self.wip.store(access_kind, non_error_val, operand, .default);
7918 return operand;
7919 }
7920 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7921 {
7922 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7923
7924 const err_int_ty = try pt.errorIntType();
7925 const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
7926 const error_offset = try errUnionErrorOffset(payload_ty, pt);
7927 // First set the non-error value.
7928 const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");
7929 _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);
7930 }
7931 // Then return the payload pointer (only if it is used).
7932 if (self.liveness.isUnused(inst)) return .none;
7933
7934 const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7935 return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
7936 }
7937
7938 fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
7939 assert(self.err_ret_trace != .none);
7940 return self.err_ret_trace;
7941 }
7942
7943 fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7944 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7945 self.err_ret_trace = try self.resolveInst(un_op);
7946 return .none;
7947 }
7948
7949 fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7950 const o = self.ng.object;
7951 const pt = self.ng.pt;
7952 const zcu = pt.zcu;
7953
7954 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7955 const struct_ty = ty_pl.ty.toType();
7956 const field_index = ty_pl.payload;
7957
7958 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
7959 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
7960 assert(self.err_ret_trace != .none);
7961 const field_ptr =
7962 try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
7963 const field_alignment = struct_ty.fieldAlignment(field_index, zcu);
7964 const field_ty = struct_ty.fieldType(field_index, zcu);
7965 const field_ptr_ty = try pt.ptrType(.{
7966 .child = field_ty.toIntern(),
7967 .flags = .{ .alignment = field_alignment },
7968 });
7969 return self.load(field_ptr, field_ptr_ty);
7970 }
7971
7972 /// As an optimization, we want to avoid unnecessary copies of
7973 /// error union/optional types when returning from a function.
7974 /// Here, we scan forward in the current block, looking to see
7975 /// if the next instruction is a return (ignoring debug instructions).
7976 ///
7977 /// The first instruction of `body_tail` is a wrap instruction.
7978 fn isNextRet(
7979 self: *FuncGen,
7980 body_tail: []const Air.Inst.Index,
7981 ) bool {
7982 const air_tags = self.air.instructions.items(.tag);
7983 for (body_tail[1..]) |body_inst| {
7984 switch (air_tags[@intFromEnum(body_inst)]) {
7985 .ret => return true,
7986 .dbg_stmt => continue,
7987 else => return false,
7988 }
7989 }
7990 // The only way to get here is to hit the end of a loop instruction
7991 // (implicit repeat).
7992 return false;
7993 }
7994
7995 fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7996 const o = self.ng.object;
7997 const pt = self.ng.pt;
7998 const zcu = pt.zcu;
7999 const inst = body_tail[0];
8000 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8001 const payload_ty = self.typeOf(ty_op.operand);
8002 const non_null_bit = try o.builder.intValue(.i8, 1);
8003 comptime assert(optional_layout_version == 3);
8004 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return non_null_bit;
8005 const operand = try self.resolveInst(ty_op.operand);
8006 const optional_ty = self.typeOfIndex(inst);
8007 if (optional_ty.optionalReprIsPayload(zcu)) return operand;
8008 const llvm_optional_ty = try o.lowerType(pt, optional_ty);
8009 if (isByRef(optional_ty, zcu)) {
8010 const directReturn = self.isNextRet(body_tail);
8011 const optional_ptr = if (directReturn)
8012 self.ret_ptr
8013 else brk: {
8014 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
8015 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
8016 break :brk optional_ptr;
8017 };
8018
8019 const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");
8020 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
8021 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
8022 const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");
8023 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
8024 return optional_ptr;
8025 }
8026 return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
8027 }
8028
8029 fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
8030 const o = self.ng.object;
8031 const pt = self.ng.pt;
8032 const zcu = pt.zcu;
8033 const inst = body_tail[0];
8034 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8035 const err_un_ty = self.typeOfIndex(inst);
8036 const operand = try self.resolveInst(ty_op.operand);
8037 const payload_ty = self.typeOf(ty_op.operand);
8038 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
8039 return operand;
8040 }
8041 const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
8042 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
8043
8044 const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
8045 const error_offset = try errUnionErrorOffset(payload_ty, pt);
8046 if (isByRef(err_un_ty, zcu)) {
8047 const directReturn = self.isNextRet(body_tail);
8048 const result_ptr = if (directReturn)
8049 self.ret_ptr
8050 else brk: {
8051 const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
8052 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
8053 break :brk result_ptr;
8054 };
8055
8056 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
8057 const err_int_ty = try pt.errorIntType();
8058 const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();
8059 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
8060 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
8061 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
8062 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
8063 return result_ptr;
8064 }
8065 var fields: [2]Builder.Value = undefined;
8066 fields[payload_offset] = operand;
8067 fields[error_offset] = ok_err_code;
8068 return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
8069 }
8070
8071 fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
8072 const o = self.ng.object;
8073 const pt = self.ng.pt;
8074 const zcu = pt.zcu;
8075 const inst = body_tail[0];
8076 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8077 const err_un_ty = self.typeOfIndex(inst);
8078 const payload_ty = err_un_ty.errorUnionPayload(zcu);
8079 const operand = try self.resolveInst(ty_op.operand);
8080 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return operand;
8081 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
8082
8083 const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
8084 const error_offset = try errUnionErrorOffset(payload_ty, pt);
8085 if (isByRef(err_un_ty, zcu)) {
8086 const directReturn = self.isNextRet(body_tail);
8087 const result_ptr = if (directReturn)
8088 self.ret_ptr
8089 else brk: {
8090 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
8091 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
8092 break :brk result_ptr;
8093 };
8094
8095 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
8096 const err_int_ty = try pt.errorIntType();
8097 const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
8098 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
8099 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
8100 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
8101 // TODO store undef to payload_ptr
8102 _ = payload_ptr;
8103 _ = payload_ptr_ty;
8104 return result_ptr;
8105 }
8106
8107 // TODO set payload bytes to undef
8108 const undef = try o.builder.undefValue(err_un_llvm_ty);
8109 return self.wip.insertValue(undef, operand, &.{error_offset}, "");
8110 }
8111
8112 fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8113 const o = self.ng.object;
8114 const pt = self.ng.pt;
8115 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8116 const index = pl_op.payload;
8117 const llvm_usize = try o.lowerType(pt, Type.usize);
8118 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
8119 try o.builder.intValue(.i32, index),
8120 }, "");
8121 }
8122
8123 fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8124 const o = self.ng.object;
8125 const pt = self.ng.pt;
8126 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8127 const index = pl_op.payload;
8128 const llvm_isize = try o.lowerType(pt, Type.isize);
8129 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
8130 try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
8131 }, "");
8132 }
8133
8134 fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8135 const o = fg.ng.object;
8136 const pt = fg.ng.pt;
8137 const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
8138 const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav);
8139 return llvm_ptr_const.toValue();
8140 }
8141
8142 fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8143 const o = self.ng.object;
8144 const pt = self.ng.pt;
8145 const zcu = pt.zcu;
8146 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8147 const lhs = try self.resolveInst(bin_op.lhs);
8148 const rhs = try self.resolveInst(bin_op.rhs);
8149 const inst_ty = self.typeOfIndex(inst);
8150 const scalar_ty = inst_ty.scalarType(zcu);
8151
8152 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
8153 return self.wip.callIntrinsic(
8154 .normal,
8155 .none,
8156 if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
8157 &.{try o.lowerType(pt, inst_ty)},
8158 &.{ lhs, rhs },
8159 "",
8160 );
8161 }
8162
8163 fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8164 const o = self.ng.object;
8165 const pt = self.ng.pt;
8166 const zcu = pt.zcu;
8167 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8168 const lhs = try self.resolveInst(bin_op.lhs);
8169 const rhs = try self.resolveInst(bin_op.rhs);
8170 const inst_ty = self.typeOfIndex(inst);
8171 const scalar_ty = inst_ty.scalarType(zcu);
8172
8173 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
8174 return self.wip.callIntrinsic(
8175 .normal,
8176 .none,
8177 if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
8178 &.{try o.lowerType(pt, inst_ty)},
8179 &.{ lhs, rhs },
8180 "",
8181 );
8182 }
8183
8184 fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8185 const o = self.ng.object;
8186 const pt = self.ng.pt;
8187 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8188 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8189 const ptr = try self.resolveInst(bin_op.lhs);
8190 const len = try self.resolveInst(bin_op.rhs);
8191 const inst_ty = self.typeOfIndex(inst);
8192 return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, "");
8193 }
8194
8195 fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8196 const zcu = self.ng.pt.zcu;
8197 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8198 const lhs = try self.resolveInst(bin_op.lhs);
8199 const rhs = try self.resolveInst(bin_op.rhs);
8200 const inst_ty = self.typeOfIndex(inst);
8201 const scalar_ty = inst_ty.scalarType(zcu);
8202
8203 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
8204 return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
8205 }
8206
8207 fn airSafeArithmetic(
8208 fg: *FuncGen,
8209 inst: Air.Inst.Index,
8210 signed_intrinsic: Builder.Intrinsic,
8211 unsigned_intrinsic: Builder.Intrinsic,
8212 ) !Builder.Value {
8213 const o = fg.ng.object;
8214 const pt = fg.ng.pt;
8215 const zcu = pt.zcu;
8216
8217 const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8218 const lhs = try fg.resolveInst(bin_op.lhs);
8219 const rhs = try fg.resolveInst(bin_op.rhs);
8220 const inst_ty = fg.typeOfIndex(inst);
8221 const scalar_ty = inst_ty.scalarType(zcu);
8222
8223 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8224 const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8225 const results =
8226 try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
8227
8228 const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
8229 const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
8230 const overflow_bit = if (overflow_bits_ty.isVector(&o.builder))
8231 try fg.wip.callIntrinsic(
8232 .normal,
8233 .none,
8234 .@"vector.reduce.or",
8235 &.{overflow_bits_ty},
8236 &.{overflow_bits},
8237 "",
8238 )
8239 else
8240 overflow_bits;
8241
8242 const fail_block = try fg.wip.block(1, "OverflowFail");
8243 const ok_block = try fg.wip.block(1, "OverflowOk");
8244 _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
8245
8246 fg.wip.cursor = .{ .block = fail_block };
8247 try fg.buildSimplePanic(.integer_overflow);
8248
8249 fg.wip.cursor = .{ .block = ok_block };
8250 return fg.wip.extractValue(results, &.{0}, "");
8251 }
8252
8253 fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8254 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8255 const lhs = try self.resolveInst(bin_op.lhs);
8256 const rhs = try self.resolveInst(bin_op.rhs);
8257
8258 return self.wip.bin(.add, lhs, rhs, "");
8259 }
8260
8261 fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8262 const o = self.ng.object;
8263 const pt = self.ng.pt;
8264 const zcu = pt.zcu;
8265 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8266 const lhs = try self.resolveInst(bin_op.lhs);
8267 const rhs = try self.resolveInst(bin_op.rhs);
8268 const inst_ty = self.typeOfIndex(inst);
8269 const scalar_ty = inst_ty.scalarType(zcu);
8270 assert(scalar_ty.zigTypeTag(zcu) == .int);
8271 return self.wip.callIntrinsic(
8272 .normal,
8273 .none,
8274 if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
8275 &.{try o.lowerType(pt, inst_ty)},
8276 &.{ lhs, rhs },
8277 "",
8278 );
8279 }
8280
8281 fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8282 const zcu = self.ng.pt.zcu;
8283 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8284 const lhs = try self.resolveInst(bin_op.lhs);
8285 const rhs = try self.resolveInst(bin_op.rhs);
8286 const inst_ty = self.typeOfIndex(inst);
8287 const scalar_ty = inst_ty.scalarType(zcu);
8288
8289 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
8290 return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
8291 }
8292
8293 fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8294 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8295 const lhs = try self.resolveInst(bin_op.lhs);
8296 const rhs = try self.resolveInst(bin_op.rhs);
8297
8298 return self.wip.bin(.sub, lhs, rhs, "");
8299 }
8300
8301 fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8302 const o = self.ng.object;
8303 const pt = self.ng.pt;
8304 const zcu = pt.zcu;
8305 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8306 const lhs = try self.resolveInst(bin_op.lhs);
8307 const rhs = try self.resolveInst(bin_op.rhs);
8308 const inst_ty = self.typeOfIndex(inst);
8309 const scalar_ty = inst_ty.scalarType(zcu);
8310 assert(scalar_ty.zigTypeTag(zcu) == .int);
8311 return self.wip.callIntrinsic(
8312 .normal,
8313 .none,
8314 if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
8315 &.{try o.lowerType(pt, inst_ty)},
8316 &.{ lhs, rhs },
8317 "",
8318 );
8319 }
8320
8321 fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8322 const zcu = self.ng.pt.zcu;
8323 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8324 const lhs = try self.resolveInst(bin_op.lhs);
8325 const rhs = try self.resolveInst(bin_op.rhs);
8326 const inst_ty = self.typeOfIndex(inst);
8327 const scalar_ty = inst_ty.scalarType(zcu);
8328
8329 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
8330 return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
8331 }
8332
8333 fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8334 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8335 const lhs = try self.resolveInst(bin_op.lhs);
8336 const rhs = try self.resolveInst(bin_op.rhs);
8337
8338 return self.wip.bin(.mul, lhs, rhs, "");
8339 }
8340
8341 fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8342 const o = self.ng.object;
8343 const pt = self.ng.pt;
8344 const zcu = pt.zcu;
8345 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8346 const lhs = try self.resolveInst(bin_op.lhs);
8347 const rhs = try self.resolveInst(bin_op.rhs);
8348 const inst_ty = self.typeOfIndex(inst);
8349 const scalar_ty = inst_ty.scalarType(zcu);
8350 assert(scalar_ty.zigTypeTag(zcu) == .int);
8351 return self.wip.callIntrinsic(
8352 .normal,
8353 .none,
8354 if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
8355 &.{try o.lowerType(pt, inst_ty)},
8356 &.{ lhs, rhs, .@"0" },
8357 "",
8358 );
8359 }
8360
8361 fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8362 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8363 const lhs = try self.resolveInst(bin_op.lhs);
8364 const rhs = try self.resolveInst(bin_op.rhs);
8365 const inst_ty = self.typeOfIndex(inst);
8366
8367 return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8368 }
8369
8370 fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8371 const zcu = self.ng.pt.zcu;
8372 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8373 const lhs = try self.resolveInst(bin_op.lhs);
8374 const rhs = try self.resolveInst(bin_op.rhs);
8375 const inst_ty = self.typeOfIndex(inst);
8376 const scalar_ty = inst_ty.scalarType(zcu);
8377
8378 if (scalar_ty.isRuntimeFloat()) {
8379 const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8380 return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
8381 }
8382 return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
8383 }
8384
8385 fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8386 const o = self.ng.object;
8387 const pt = self.ng.pt;
8388 const zcu = pt.zcu;
8389 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8390 const lhs = try self.resolveInst(bin_op.lhs);
8391 const rhs = try self.resolveInst(bin_op.rhs);
8392 const inst_ty = self.typeOfIndex(inst);
8393 const scalar_ty = inst_ty.scalarType(zcu);
8394
8395 if (scalar_ty.isRuntimeFloat()) {
8396 const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8397 return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
8398 }
8399 if (scalar_ty.isSignedInt(zcu)) {
8400 const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8401
8402 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8403 var stack align(@max(
8404 @alignOf(std.heap.StackFallbackAllocator(0)),
8405 @alignOf(ExpectedContents),
8406 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8407 const allocator = stack.get();
8408
8409 const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8410 var smin_big_int: std.math.big.int.Mutable = .{
8411 .limbs = try allocator.alloc(
8412 std.math.big.Limb,
8413 std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8414 ),
8415 .len = undefined,
8416 .positive = undefined,
8417 };
8418 defer allocator.free(smin_big_int.limbs);
8419 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8420 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8421 inst_llvm_ty.scalarType(&o.builder),
8422 smin_big_int.toConst(),
8423 ));
8424
8425 const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
8426 const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
8427 const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
8428 const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
8429 const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
8430 const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
8431 return self.wip.bin(.@"add nsw", div, correction, "divFloor");
8432 }
8433 return self.wip.bin(.udiv, lhs, rhs, "");
8434 }
8435
8436 fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8437 const zcu = self.ng.pt.zcu;
8438 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8439 const lhs = try self.resolveInst(bin_op.lhs);
8440 const rhs = try self.resolveInst(bin_op.rhs);
8441 const inst_ty = self.typeOfIndex(inst);
8442 const scalar_ty = inst_ty.scalarType(zcu);
8443
8444 if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8445 return self.wip.bin(
8446 if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
8447 lhs,
8448 rhs,
8449 "",
8450 );
8451 }
8452
8453 fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8454 const zcu = self.ng.pt.zcu;
8455 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8456 const lhs = try self.resolveInst(bin_op.lhs);
8457 const rhs = try self.resolveInst(bin_op.rhs);
8458 const inst_ty = self.typeOfIndex(inst);
8459 const scalar_ty = inst_ty.scalarType(zcu);
8460
8461 if (scalar_ty.isRuntimeFloat())
8462 return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8463 return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
8464 .srem
8465 else
8466 .urem, lhs, rhs, "");
8467 }
8468
8469 fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8470 const o = self.ng.object;
8471 const pt = self.ng.pt;
8472 const zcu = pt.zcu;
8473 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8474 const lhs = try self.resolveInst(bin_op.lhs);
8475 const rhs = try self.resolveInst(bin_op.rhs);
8476 const inst_ty = self.typeOfIndex(inst);
8477 const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8478 const scalar_ty = inst_ty.scalarType(zcu);
8479
8480 if (scalar_ty.isRuntimeFloat()) {
8481 const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8482 const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
8483 const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
8484 const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8485 const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero });
8486 return self.wip.select(fast, ltz, c, a, "");
8487 }
8488 if (scalar_ty.isSignedInt(zcu)) {
8489 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8490 var stack align(@max(
8491 @alignOf(std.heap.StackFallbackAllocator(0)),
8492 @alignOf(ExpectedContents),
8493 )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8494 const allocator = stack.get();
8495
8496 const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8497 var smin_big_int: std.math.big.int.Mutable = .{
8498 .limbs = try allocator.alloc(
8499 std.math.big.Limb,
8500 std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8501 ),
8502 .len = undefined,
8503 .positive = undefined,
8504 };
8505 defer allocator.free(smin_big_int.limbs);
8506 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8507 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8508 inst_llvm_ty.scalarType(&o.builder),
8509 smin_big_int.toConst(),
8510 ));
8511
8512 const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
8513 const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
8514 const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
8515 const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
8516 const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8517 const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
8518 return self.wip.bin(.@"add nsw", correction, rem, "mod");
8519 }
8520 return self.wip.bin(.urem, lhs, rhs, "");
8521 }
8522
8523 fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8524 const o = self.ng.object;
8525 const pt = self.ng.pt;
8526 const zcu = pt.zcu;
8527 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8528 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8529 const ptr = try self.resolveInst(bin_op.lhs);
8530 const offset = try self.resolveInst(bin_op.rhs);
8531 const ptr_ty = self.typeOf(bin_op.lhs);
8532 const llvm_elem_ty = try o.lowerPtrElemTy(pt, ptr_ty.childType(zcu));
8533 switch (ptr_ty.ptrSize(zcu)) {
8534 // It's a pointer to an array, so according to LLVM we need an extra GEP index.
8535 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8536 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,
8537 }, ""),
8538 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
8539 .slice => {
8540 const base = try self.wip.extractValue(ptr, &.{0}, "");
8541 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
8542 },
8543 }
8544 }
8545
8546 fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8547 const o = self.ng.object;
8548 const pt = self.ng.pt;
8549 const zcu = pt.zcu;
8550 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8551 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8552 const ptr = try self.resolveInst(bin_op.lhs);
8553 const offset = try self.resolveInst(bin_op.rhs);
8554 const negative_offset = try self.wip.neg(offset, "");
8555 const ptr_ty = self.typeOf(bin_op.lhs);
8556 const llvm_elem_ty = try o.lowerPtrElemTy(pt, ptr_ty.childType(zcu));
8557 switch (ptr_ty.ptrSize(zcu)) {
8558 // It's a pointer to an array, so according to LLVM we need an extra GEP index.
8559 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8560 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,
8561 }, ""),
8562 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),
8563 .slice => {
8564 const base = try self.wip.extractValue(ptr, &.{0}, "");
8565 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
8566 },
8567 }
8568 }
8569
8570 fn airOverflow(
8571 self: *FuncGen,
8572 inst: Air.Inst.Index,
8573 signed_intrinsic: Builder.Intrinsic,
8574 unsigned_intrinsic: Builder.Intrinsic,
8575 ) !Builder.Value {
8576 const o = self.ng.object;
8577 const pt = self.ng.pt;
8578 const zcu = pt.zcu;
8579 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8580 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8581
8582 const lhs = try self.resolveInst(extra.lhs);
8583 const rhs = try self.resolveInst(extra.rhs);
8584
8585 const lhs_ty = self.typeOf(extra.lhs);
8586 const scalar_ty = lhs_ty.scalarType(zcu);
8587 const inst_ty = self.typeOfIndex(inst);
8588
8589 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8590 const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8591 const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
8592 const results =
8593 try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
8594
8595 const result_val = try self.wip.extractValue(results, &.{0}, "");
8596 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
8597
8598 const result_index = o.llvmFieldIndex(inst_ty, 0).?;
8599 const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
8600
8601 if (isByRef(inst_ty, zcu)) {
8602 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
8603 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
8604 {
8605 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
8606 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
8607 }
8608 {
8609 const overflow_alignment = comptime Builder.Alignment.fromByteUnits(1);
8610 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
8611 _ = try self.wip.store(.normal, overflow_bit, field_ptr, overflow_alignment);
8612 }
8613
8614 return alloca_inst;
8615 }
8616
8617 var fields: [2]Builder.Value = undefined;
8618 fields[result_index] = result_val;
8619 fields[overflow_index] = overflow_bit;
8620 return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
8621 }
8622
8623 fn buildElementwiseCall(
8624 self: *FuncGen,
8625 llvm_fn: Builder.Function.Index,
8626 args_vectors: []const Builder.Value,
8627 result_vector: Builder.Value,
8628 vector_len: usize,
8629 ) !Builder.Value {
8630 const o = self.ng.object;
8631 assert(args_vectors.len <= 3);
8632
8633 var i: usize = 0;
8634 var result = result_vector;
8635 while (i < vector_len) : (i += 1) {
8636 const index_i32 = try o.builder.intValue(.i32, i);
8637
8638 var args: [3]Builder.Value = undefined;
8639 for (args[0..args_vectors.len], args_vectors) |*arg_elem, arg_vector| {
8640 arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, "");
8641 }
8642 const result_elem = try self.wip.call(
8643 .normal,
8644 .ccc,
8645 .none,
8646 llvm_fn.typeOf(&o.builder),
8647 llvm_fn.toValue(&o.builder),
8648 args[0..args_vectors.len],
8649 "",
8650 );
8651 result = try self.wip.insertElement(result, result_elem, index_i32, "");
8652 }
8653 return result;
8654 }
8655
8656 fn getLibcFunction(
8657 self: *FuncGen,
8658 fn_name: Builder.StrtabString,
8659 param_types: []const Builder.Type,
8660 return_type: Builder.Type,
8661 ) Allocator.Error!Builder.Function.Index {
8662 const o = self.ng.object;
8663 if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) {
8664 .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
8665 .function => |function| function,
8666 .variable, .replaced => unreachable,
8667 };
8668 return o.builder.addFunction(
8669 try o.builder.fnType(return_type, param_types, .normal),
8670 fn_name,
8671 toLlvmAddressSpace(.generic, self.ng.pt.zcu.getTarget()),
8672 );
8673 }
8674
8675 /// Creates a floating point comparison by lowering to the appropriate
8676 /// hardware instruction or softfloat routine for the target
8677 fn buildFloatCmp(
8678 self: *FuncGen,
8679 fast: Builder.FastMathKind,
8680 pred: math.CompareOperator,
8681 ty: Type,
8682 params: [2]Builder.Value,
8683 ) !Builder.Value {
8684 const o = self.ng.object;
8685 const pt = self.ng.pt;
8686 const zcu = pt.zcu;
8687 const target = zcu.getTarget();
8688 const scalar_ty = ty.scalarType(zcu);
8689 const scalar_llvm_ty = try o.lowerType(pt, scalar_ty);
8690
8691 if (intrinsicsAllowed(scalar_ty, target)) {
8692 const cond: Builder.FloatCondition = switch (pred) {
8693 .eq => .oeq,
8694 .neq => .une,
8695 .lt => .olt,
8696 .lte => .ole,
8697 .gt => .ogt,
8698 .gte => .oge,
8699 };
8700 return self.wip.fcmp(fast, cond, params[0], params[1], "");
8701 }
8702
8703 const float_bits = scalar_ty.floatBits(target);
8704 const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits);
8705 const fn_base_name = switch (pred) {
8706 .neq => "ne",
8707 .eq => "eq",
8708 .lt => "lt",
8709 .lte => "le",
8710 .gt => "gt",
8711 .gte => "ge",
8712 };
8713 const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev });
8714
8715 const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32);
8716
8717 const int_cond: Builder.IntegerCondition = switch (pred) {
8718 .eq => .eq,
8719 .neq => .ne,
8720 .lt => .slt,
8721 .lte => .sle,
8722 .gt => .sgt,
8723 .gte => .sge,
8724 };
8725
8726 if (ty.zigTypeTag(zcu) == .vector) {
8727 const vec_len = ty.vectorLen(zcu);
8728 const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32);
8729
8730 const init = try o.builder.poisonValue(vector_result_ty);
8731 const result = try self.buildElementwiseCall(libc_fn, ¶ms, init, vec_len);
8732
8733 const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0");
8734 return self.wip.icmp(int_cond, result, zero_vector, "");
8735 }
8736
8737 const result = try self.wip.call(
8738 .normal,
8739 .ccc,
8740 .none,
8741 libc_fn.typeOf(&o.builder),
8742 libc_fn.toValue(&o.builder),
8743 ¶ms,
8744 "",
8745 );
8746 return self.wip.icmp(int_cond, result, .@"0", "");
8747 }
8748
8749 const FloatOp = enum {
8750 add,
8751 ceil,
8752 cos,
8753 div,
8754 exp,
8755 exp2,
8756 fabs,
8757 floor,
8758 fma,
8759 fmax,
8760 fmin,
8761 fmod,
8762 log,
8763 log10,
8764 log2,
8765 mul,
8766 neg,
8767 round,
8768 sin,
8769 sqrt,
8770 sub,
8771 tan,
8772 trunc,
8773 };
8774
8775 const FloatOpStrat = union(enum) {
8776 intrinsic: []const u8,
8777 libc: Builder.String,
8778 };
8779
8780 /// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
8781 /// by lowering to the appropriate hardware instruction or softfloat
8782 /// routine for the target
8783 fn buildFloatOp(
8784 self: *FuncGen,
8785 comptime op: FloatOp,
8786 fast: Builder.FastMathKind,
8787 ty: Type,
8788 comptime params_len: usize,
8789 params: [params_len]Builder.Value,
8790 ) !Builder.Value {
8791 const o = self.ng.object;
8792 const pt = self.ng.pt;
8793 const zcu = pt.zcu;
8794 const target = zcu.getTarget();
8795 const scalar_ty = ty.scalarType(zcu);
8796 const llvm_ty = try o.lowerType(pt, ty);
8797
8798 if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
8799 // Some operations are dedicated LLVM instructions, not available as intrinsics
8800 .neg => return self.wip.un(.fneg, params[0], ""),
8801 .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) {
8802 .normal => switch (op) {
8803 .add => .fadd,
8804 .sub => .fsub,
8805 .mul => .fmul,
8806 .div => .fdiv,
8807 .fmod => .frem,
8808 else => unreachable,
8809 },
8810 .fast => switch (op) {
8811 .add => .@"fadd fast",
8812 .sub => .@"fsub fast",
8813 .mul => .@"fmul fast",
8814 .div => .@"fdiv fast",
8815 .fmod => .@"frem fast",
8816 else => unreachable,
8817 },
8818 }, params[0], params[1], ""),
8819 .fmax,
8820 .fmin,
8821 .ceil,
8822 .cos,
8823 .exp,
8824 .exp2,
8825 .fabs,
8826 .floor,
8827 .log,
8828 .log10,
8829 .log2,
8830 .round,
8831 .sin,
8832 .sqrt,
8833 .trunc,
8834 .fma,
8835 => return self.wip.callIntrinsic(fast, .none, switch (op) {
8836 .fmax => .maxnum,
8837 .fmin => .minnum,
8838 .ceil => .ceil,
8839 .cos => .cos,
8840 .exp => .exp,
8841 .exp2 => .exp2,
8842 .fabs => .fabs,
8843 .floor => .floor,
8844 .log => .log,
8845 .log10 => .log10,
8846 .log2 => .log2,
8847 .round => .round,
8848 .sin => .sin,
8849 .sqrt => .sqrt,
8850 .trunc => .trunc,
8851 .fma => .fma,
8852 else => unreachable,
8853 }, &.{llvm_ty}, ¶ms, ""),
8854 .tan => unreachable,
8855 };
8856
8857 const float_bits = scalar_ty.floatBits(target);
8858 const fn_name = switch (op) {
8859 .neg => {
8860 // In this case we can generate a softfloat negation by XORing the
8861 // bits with a constant.
8862 const int_ty = try o.builder.intType(@intCast(float_bits));
8863 const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder);
8864 const sign_mask = try o.builder.splatValue(
8865 cast_ty,
8866 try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)),
8867 );
8868 const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, "");
8869 const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, "");
8870 return self.wip.cast(.bitcast, result, llvm_ty, "");
8871 },
8872 .add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
8873 @tagName(op), compilerRtFloatAbbrev(float_bits),
8874 }),
8875 .ceil,
8876 .cos,
8877 .exp,
8878 .exp2,
8879 .fabs,
8880 .floor,
8881 .fma,
8882 .fmax,
8883 .fmin,
8884 .fmod,
8885 .log,
8886 .log10,
8887 .log2,
8888 .round,
8889 .sin,
8890 .sqrt,
8891 .tan,
8892 .trunc,
8893 => try o.builder.strtabStringFmt("{s}{s}{s}", .{
8894 libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
8895 }),
8896 };
8897
8898 const scalar_llvm_ty = llvm_ty.scalarType(&o.builder);
8899 const libc_fn = try self.getLibcFunction(
8900 fn_name,
8901 ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len],
8902 scalar_llvm_ty,
8903 );
8904 if (ty.zigTypeTag(zcu) == .vector) {
8905 const result = try o.builder.poisonValue(llvm_ty);
8906 return self.buildElementwiseCall(libc_fn, ¶ms, result, ty.vectorLen(zcu));
8907 }
8908
8909 return self.wip.call(
8910 fast.toCallKind(),
8911 .ccc,
8912 .none,
8913 libc_fn.typeOf(&o.builder),
8914 libc_fn.toValue(&o.builder),
8915 ¶ms,
8916 "",
8917 );
8918 }
8919
8920 fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8921 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8922 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
8923
8924 const mulend1 = try self.resolveInst(extra.lhs);
8925 const mulend2 = try self.resolveInst(extra.rhs);
8926 const addend = try self.resolveInst(pl_op.operand);
8927
8928 const ty = self.typeOfIndex(inst);
8929 return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
8930 }
8931
8932 fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8933 const o = self.ng.object;
8934 const pt = self.ng.pt;
8935 const zcu = pt.zcu;
8936 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8937 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8938
8939 const lhs = try self.resolveInst(extra.lhs);
8940 const rhs = try self.resolveInst(extra.rhs);
8941
8942 const lhs_ty = self.typeOf(extra.lhs);
8943 if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu))
8944 return self.ng.todo("implement vector shifts with scalar rhs", .{});
8945 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8946
8947 const dest_ty = self.typeOfIndex(inst);
8948 const llvm_dest_ty = try o.lowerType(pt, dest_ty);
8949
8950 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8951
8952 const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
8953 const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
8954 .ashr
8955 else
8956 .lshr, result, casted_rhs, "");
8957
8958 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
8959
8960 const result_index = o.llvmFieldIndex(dest_ty, 0).?;
8961 const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
8962
8963 if (isByRef(dest_ty, zcu)) {
8964 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
8965 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
8966 {
8967 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
8968 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);
8969 }
8970 {
8971 const field_alignment = comptime Builder.Alignment.fromByteUnits(1);
8972 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
8973 _ = try self.wip.store(.normal, overflow_bit, field_ptr, field_alignment);
8974 }
8975 return alloca_inst;
8976 }
8977
8978 var fields: [2]Builder.Value = undefined;
8979 fields[result_index] = result;
8980 fields[overflow_index] = overflow_bit;
8981 return self.wip.buildAggregate(llvm_dest_ty, &fields, "");
8982 }
8983
8984 fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8985 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8986 const lhs = try self.resolveInst(bin_op.lhs);
8987 const rhs = try self.resolveInst(bin_op.rhs);
8988 return self.wip.bin(.@"and", lhs, rhs, "");
8989 }
8990
8991 fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8992 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8993 const lhs = try self.resolveInst(bin_op.lhs);
8994 const rhs = try self.resolveInst(bin_op.rhs);
8995 return self.wip.bin(.@"or", lhs, rhs, "");
8996 }
8997
8998 fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8999 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9000 const lhs = try self.resolveInst(bin_op.lhs);
9001 const rhs = try self.resolveInst(bin_op.rhs);
9002 return self.wip.bin(.xor, lhs, rhs, "");
9003 }
9004
9005 fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9006 const o = self.ng.object;
9007 const pt = self.ng.pt;
9008 const zcu = pt.zcu;
9009 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9010
9011 const lhs = try self.resolveInst(bin_op.lhs);
9012 const rhs = try self.resolveInst(bin_op.rhs);
9013
9014 const lhs_ty = self.typeOf(bin_op.lhs);
9015 if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
9016 return self.ng.todo("implement vector shifts with scalar rhs", .{});
9017 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
9018
9019 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
9020 return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
9021 .@"shl nsw"
9022 else
9023 .@"shl nuw", lhs, casted_rhs, "");
9024 }
9025
9026 fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9027 const o = self.ng.object;
9028 const pt = self.ng.pt;
9029 const zcu = pt.zcu;
9030 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9031
9032 const lhs = try self.resolveInst(bin_op.lhs);
9033 const rhs = try self.resolveInst(bin_op.rhs);
9034
9035 const lhs_ty = self.typeOf(bin_op.lhs);
9036 if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
9037 return self.ng.todo("implement vector shifts with scalar rhs", .{});
9038
9039 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
9040 return self.wip.bin(.shl, lhs, casted_rhs, "");
9041 }
9042
9043 fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9044 const o = self.ng.object;
9045 const pt = self.ng.pt;
9046 const zcu = pt.zcu;
9047 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9048
9049 const lhs = try self.resolveInst(bin_op.lhs);
9050 const rhs = try self.resolveInst(bin_op.rhs);
9051
9052 const lhs_ty = self.typeOf(bin_op.lhs);
9053 const lhs_info = lhs_ty.intInfo(zcu);
9054 const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
9055 const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder);
9056
9057 const rhs_ty = self.typeOf(bin_op.rhs);
9058 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu))
9059 return self.ng.todo("implement vector shifts with scalar rhs", .{});
9060 const rhs_info = rhs_ty.intInfo(zcu);
9061 assert(rhs_info.signedness == .unsigned);
9062 const llvm_rhs_ty = try o.lowerType(pt, rhs_ty);
9063 const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder);
9064
9065 const result = try self.wip.callIntrinsic(
9066 .normal,
9067 .none,
9068 switch (lhs_info.signedness) {
9069 .signed => .@"sshl.sat",
9070 .unsigned => .@"ushl.sat",
9071 },
9072 &.{llvm_lhs_ty},
9073 &.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
9074 "",
9075 );
9076
9077 // LLVM langref says "If b is (statically or dynamically) equal to or
9078 // larger than the integer bit width of the arguments, the result is a
9079 // poison value."
9080 // However Zig semantics says that saturating shift left can never produce
9081 // undefined; instead it saturates.
9082 if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
9083 const bits = try o.builder.splatValue(
9084 llvm_rhs_ty,
9085 try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
9086 );
9087 const in_range = try self.wip.icmp(.ult, rhs, bits, "");
9088 const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
9089 .signed => {
9090 const zero = try o.builder.splatValue(
9091 llvm_lhs_ty,
9092 try o.builder.intConst(llvm_lhs_scalar_ty, 0),
9093 );
9094 const smin = try o.builder.splatValue(
9095 llvm_lhs_ty,
9096 try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
9097 );
9098 const smax = try o.builder.splatValue(
9099 llvm_lhs_ty,
9100 try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
9101 );
9102 const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
9103 const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
9104 const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
9105 break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
9106 },
9107 .unsigned => {
9108 const zero = try o.builder.splatValue(
9109 llvm_lhs_ty,
9110 try o.builder.intConst(llvm_lhs_scalar_ty, 0),
9111 );
9112 const umax = try o.builder.splatValue(
9113 llvm_lhs_ty,
9114 try o.builder.intConst(llvm_lhs_scalar_ty, -1),
9115 );
9116 const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
9117 break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
9118 },
9119 };
9120 return self.wip.select(.normal, in_range, result, lhs_sat, "");
9121 }
9122
9123 fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value {
9124 const o = self.ng.object;
9125 const pt = self.ng.pt;
9126 const zcu = pt.zcu;
9127 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9128
9129 const lhs = try self.resolveInst(bin_op.lhs);
9130 const rhs = try self.resolveInst(bin_op.rhs);
9131
9132 const lhs_ty = self.typeOf(bin_op.lhs);
9133 if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
9134 return self.ng.todo("implement vector shifts with scalar rhs", .{});
9135 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
9136
9137 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
9138 const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
9139
9140 return self.wip.bin(if (is_exact)
9141 if (is_signed_int) .@"ashr exact" else .@"lshr exact"
9142 else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
9143 }
9144
9145 fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9146 const o = self.ng.object;
9147 const pt = self.ng.pt;
9148 const zcu = pt.zcu;
9149 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9150 const operand = try self.resolveInst(ty_op.operand);
9151 const operand_ty = self.typeOf(ty_op.operand);
9152 const scalar_ty = operand_ty.scalarType(zcu);
9153
9154 switch (scalar_ty.zigTypeTag(zcu)) {
9155 .int => return self.wip.callIntrinsic(
9156 .normal,
9157 .none,
9158 .abs,
9159 &.{try o.lowerType(pt, operand_ty)},
9160 &.{ operand, try o.builder.intValue(.i1, 0) },
9161 "",
9162 ),
9163 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
9164 else => unreachable,
9165 }
9166 }
9167
9168 fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9169 const o = fg.ng.object;
9170 const pt = fg.ng.pt;
9171 const zcu = pt.zcu;
9172 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9173 const dest_ty = fg.typeOfIndex(inst);
9174 const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9175 const operand = try fg.resolveInst(ty_op.operand);
9176 const operand_ty = fg.typeOf(ty_op.operand);
9177 const operand_info = operand_ty.intInfo(zcu);
9178
9179 const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
9180
9181 bounds_check: {
9182 const dest_scalar = dest_ty.scalarType(zcu);
9183 const operand_scalar = operand_ty.scalarType(zcu);
9184
9185 const dest_info = dest_ty.intInfo(zcu);
9186
9187 const have_min_check, const have_max_check = c: {
9188 const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
9189 const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
9190
9191 const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
9192 const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
9193
9194 break :c .{
9195 operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
9196 dest_pos_bits < operand_pos_bits,
9197 };
9198 };
9199
9200 if (!have_min_check and !have_max_check) break :bounds_check;
9201
9202 const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9203 const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar);
9204
9205 const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
9206 assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
9207
9208 const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
9209
9210 if (have_min_check) {
9211 const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9212 const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
9213 const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
9214 const ok = if (is_vector) ok: {
9215 const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9216 break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9217 } else ok_maybe_vec;
9218 if (safety) {
9219 const fail_block = try fg.wip.block(1, "IntMinFail");
9220 const ok_block = try fg.wip.block(1, "IntMinOk");
9221 _ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9222 fg.wip.cursor = .{ .block = fail_block };
9223 try fg.buildSimplePanic(panic_id);
9224 fg.wip.cursor = .{ .block = ok_block };
9225 } else {
9226 _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9227 }
9228 }
9229
9230 if (have_max_check) {
9231 const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9232 const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
9233 const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
9234 const ok = if (is_vector) ok: {
9235 const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9236 break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9237 } else ok_maybe_vec;
9238 if (safety) {
9239 const fail_block = try fg.wip.block(1, "IntMaxFail");
9240 const ok_block = try fg.wip.block(1, "IntMaxOk");
9241 _ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9242 fg.wip.cursor = .{ .block = fail_block };
9243 try fg.buildSimplePanic(panic_id);
9244 fg.wip.cursor = .{ .block = ok_block };
9245 } else {
9246 _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9247 }
9248 }
9249 }
9250
9251 const result = try fg.wip.conv(switch (operand_info.signedness) {
9252 .signed => .signed,
9253 .unsigned => .unsigned,
9254 }, operand, dest_llvm_ty, "");
9255
9256 if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
9257 const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty);
9258 const is_valid_enum_val = try fg.wip.call(
9259 .normal,
9260 .fastcc,
9261 .none,
9262 llvm_fn.typeOf(&o.builder),
9263 llvm_fn.toValue(&o.builder),
9264 &.{result},
9265 "",
9266 );
9267 const fail_block = try fg.wip.block(1, "ValidEnumFail");
9268 const ok_block = try fg.wip.block(1, "ValidEnumOk");
9269 _ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
9270 fg.wip.cursor = .{ .block = fail_block };
9271 try fg.buildSimplePanic(.invalid_enum_value);
9272 fg.wip.cursor = .{ .block = ok_block };
9273 }
9274
9275 return result;
9276 }
9277
9278 fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9279 const o = self.ng.object;
9280 const pt = self.ng.pt;
9281 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9282 const operand = try self.resolveInst(ty_op.operand);
9283 const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
9284 return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
9285 }
9286
9287 fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9288 const o = self.ng.object;
9289 const pt = self.ng.pt;
9290 const zcu = pt.zcu;
9291 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9292 const operand = try self.resolveInst(ty_op.operand);
9293 const operand_ty = self.typeOf(ty_op.operand);
9294 const dest_ty = self.typeOfIndex(inst);
9295 const target = zcu.getTarget();
9296
9297 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9298 return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), "");
9299 } else {
9300 const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9301 const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9302
9303 const dest_bits = dest_ty.floatBits(target);
9304 const src_bits = operand_ty.floatBits(target);
9305 const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
9306 compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9307 });
9308
9309 const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9310 return self.wip.call(
9311 .normal,
9312 .ccc,
9313 .none,
9314 libc_fn.typeOf(&o.builder),
9315 libc_fn.toValue(&o.builder),
9316 &.{operand},
9317 "",
9318 );
9319 }
9320 }
9321
9322 fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9323 const o = self.ng.object;
9324 const pt = self.ng.pt;
9325 const zcu = pt.zcu;
9326 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9327 const operand = try self.resolveInst(ty_op.operand);
9328 const operand_ty = self.typeOf(ty_op.operand);
9329 const dest_ty = self.typeOfIndex(inst);
9330 const target = zcu.getTarget();
9331
9332 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9333 return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), "");
9334 } else {
9335 const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9336 const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9337
9338 const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
9339 const src_bits = operand_ty.scalarType(zcu).floatBits(target);
9340 const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
9341 compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9342 });
9343
9344 const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9345 if (dest_ty.isVector(zcu)) return self.buildElementwiseCall(
9346 libc_fn,
9347 &.{operand},
9348 try o.builder.poisonValue(dest_llvm_ty),
9349 dest_ty.vectorLen(zcu),
9350 );
9351 return self.wip.call(
9352 .normal,
9353 .ccc,
9354 .none,
9355 libc_fn.typeOf(&o.builder),
9356 libc_fn.toValue(&o.builder),
9357 &.{operand},
9358 "",
9359 );
9360 }
9361 }
9362
9363 fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9364 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9365 const operand_ty = self.typeOf(ty_op.operand);
9366 const inst_ty = self.typeOfIndex(inst);
9367 const operand = try self.resolveInst(ty_op.operand);
9368 return self.bitCast(operand, operand_ty, inst_ty);
9369 }
9370
9371 fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value {
9372 const o = self.ng.object;
9373 const pt = self.ng.pt;
9374 const zcu = pt.zcu;
9375 const operand_is_ref = isByRef(operand_ty, zcu);
9376 const result_is_ref = isByRef(inst_ty, zcu);
9377 const llvm_dest_ty = try o.lowerType(pt, inst_ty);
9378
9379 if (operand_is_ref and result_is_ref) {
9380 // They are both pointers, so just return the same opaque pointer :)
9381 return operand;
9382 }
9383
9384 if (llvm_dest_ty.isInteger(&o.builder) and
9385 operand.typeOfWip(&self.wip).isInteger(&o.builder))
9386 {
9387 return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
9388 }
9389
9390 const operand_scalar_ty = operand_ty.scalarType(zcu);
9391 const inst_scalar_ty = inst_ty.scalarType(zcu);
9392 if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
9393 return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
9394 }
9395 if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
9396 return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
9397 }
9398
9399 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
9400 const elem_ty = operand_ty.childType(zcu);
9401 if (!result_is_ref) {
9402 return self.ng.todo("implement bitcast vector to non-ref array", .{});
9403 }
9404 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9405 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9406 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9407 if (bitcast_ok) {
9408 _ = try self.wip.store(.normal, operand, array_ptr, alignment);
9409 } else {
9410 // If the ABI size of the element type is not evenly divisible by size in bits;
9411 // a simple bitcast will not work, and we fall back to extractelement.
9412 const llvm_usize = try o.lowerType(pt, Type.usize);
9413 const usize_zero = try o.builder.intValue(llvm_usize, 0);
9414 const vector_len = operand_ty.arrayLen(zcu);
9415 var i: u64 = 0;
9416 while (i < vector_len) : (i += 1) {
9417 const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{
9418 usize_zero, try o.builder.intValue(llvm_usize, i),
9419 }, "");
9420 const elem =
9421 try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
9422 _ = try self.wip.store(.normal, elem, elem_ptr, .default);
9423 }
9424 }
9425 return array_ptr;
9426 } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
9427 const elem_ty = operand_ty.childType(zcu);
9428 const llvm_vector_ty = try o.lowerType(pt, inst_ty);
9429 if (!operand_is_ref) return self.ng.todo("implement bitcast non-ref array to vector", .{});
9430
9431 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9432 if (bitcast_ok) {
9433 // The array is aligned to the element's alignment, while the vector might have a completely
9434 // different alignment. This means we need to enforce the alignment of this load.
9435 const alignment = elem_ty.abiAlignment(zcu).toLlvm();
9436 return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
9437 } else {
9438 // If the ABI size of the element type is not evenly divisible by size in bits;
9439 // a simple bitcast will not work, and we fall back to extractelement.
9440 const array_llvm_ty = try o.lowerType(pt, operand_ty);
9441 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9442 const llvm_usize = try o.lowerType(pt, Type.usize);
9443 const usize_zero = try o.builder.intValue(llvm_usize, 0);
9444 const vector_len = operand_ty.arrayLen(zcu);
9445 var vector = try o.builder.poisonValue(llvm_vector_ty);
9446 var i: u64 = 0;
9447 while (i < vector_len) : (i += 1) {
9448 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{
9449 usize_zero, try o.builder.intValue(llvm_usize, i),
9450 }, "");
9451 const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
9452 vector =
9453 try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
9454 }
9455 return vector;
9456 }
9457 }
9458
9459 if (operand_is_ref) {
9460 const alignment = operand_ty.abiAlignment(zcu).toLlvm();
9461 return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
9462 }
9463
9464 if (result_is_ref) {
9465 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9466 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9467 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
9468 return result_ptr;
9469 }
9470
9471 if (llvm_dest_ty.isStruct(&o.builder) or
9472 ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
9473 operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
9474 {
9475 // Both our operand and our result are values, not pointers,
9476 // but LLVM won't let us bitcast struct values or vectors with padding bits.
9477 // Therefore, we store operand to alloca, then load for result.
9478 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9479 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9480 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
9481 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
9482 }
9483
9484 return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
9485 }
9486
9487 fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9488 const o = self.ng.object;
9489 const pt = self.ng.pt;
9490 const zcu = pt.zcu;
9491 const arg_val = self.args[self.arg_index];
9492 self.arg_index += 1;
9493
9494 // llvm does not support debug info for naked function arguments
9495 if (self.is_naked) return arg_val;
9496
9497 const inst_ty = self.typeOfIndex(inst);
9498
9499 const func = zcu.funcInfo(zcu.navValue(self.ng.nav_index).toIntern());
9500 const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
9501 const file = zcu.fileByIndex(func_zir.file);
9502
9503 const mod = file.mod.?;
9504 if (mod.strip) return arg_val;
9505 const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
9506 const zir = &file.zir.?;
9507 const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
9508
9509 const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
9510 const lbrace_col = func.lbrace_column + 1;
9511
9512 const debug_parameter = try o.builder.debugParameter(
9513 if (name.len > 0) try o.builder.metadataString(name) else null,
9514 self.file,
9515 self.scope,
9516 lbrace_line,
9517 try o.lowerDebugType(pt, inst_ty),
9518 self.arg_index,
9519 );
9520
9521 const old_location = self.wip.debug_location;
9522 self.wip.debug_location = .{ .location = .{
9523 .line = lbrace_line,
9524 .column = lbrace_col,
9525 .scope = self.scope.toOptional(),
9526 .inlined_at = .none,
9527 } };
9528
9529 if (isByRef(inst_ty, zcu)) {
9530 _ = try self.wip.callIntrinsic(
9531 .normal,
9532 .none,
9533 .@"dbg.declare",
9534 &.{},
9535 &.{
9536 (try self.wip.debugValue(arg_val)).toValue(),
9537 debug_parameter.toValue(),
9538 (try o.builder.debugExpression(&.{})).toValue(),
9539 },
9540 "",
9541 );
9542 } else if (mod.optimize_mode == .Debug) {
9543 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9544 const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
9545 _ = try self.wip.store(.normal, arg_val, alloca, alignment);
9546 _ = try self.wip.callIntrinsic(
9547 .normal,
9548 .none,
9549 .@"dbg.declare",
9550 &.{},
9551 &.{
9552 (try self.wip.debugValue(alloca)).toValue(),
9553 debug_parameter.toValue(),
9554 (try o.builder.debugExpression(&.{})).toValue(),
9555 },
9556 "",
9557 );
9558 } else {
9559 _ = try self.wip.callIntrinsic(
9560 .normal,
9561 .none,
9562 .@"dbg.value",
9563 &.{},
9564 &.{
9565 (try self.wip.debugValue(arg_val)).toValue(),
9566 debug_parameter.toValue(),
9567 (try o.builder.debugExpression(&.{})).toValue(),
9568 },
9569 "",
9570 );
9571 }
9572
9573 self.wip.debug_location = old_location;
9574 return arg_val;
9575 }
9576
9577 fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9578 const o = self.ng.object;
9579 const pt = self.ng.pt;
9580 const zcu = pt.zcu;
9581 const ptr_ty = self.typeOfIndex(inst);
9582 const pointee_type = ptr_ty.childType(zcu);
9583 if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime(zcu))
9584 return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9585
9586 const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
9587 const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9588 return self.buildAlloca(pointee_llvm_ty, alignment);
9589 }
9590
9591 fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9592 const o = self.ng.object;
9593 const pt = self.ng.pt;
9594 const zcu = pt.zcu;
9595 const ptr_ty = self.typeOfIndex(inst);
9596 const ret_ty = ptr_ty.childType(zcu);
9597 if (!ret_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu))
9598 return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9599 if (self.ret_ptr != .none) return self.ret_ptr;
9600 const ret_llvm_ty = try o.lowerType(pt, ret_ty);
9601 const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9602 return self.buildAlloca(ret_llvm_ty, alignment);
9603 }
9604
9605 /// Use this instead of builder.buildAlloca, because this function makes sure to
9606 /// put the alloca instruction at the top of the function!
9607 fn buildAlloca(
9608 self: *FuncGen,
9609 llvm_ty: Builder.Type,
9610 alignment: Builder.Alignment,
9611 ) Allocator.Error!Builder.Value {
9612 const target = self.ng.pt.zcu.getTarget();
9613 return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
9614 }
9615
9616 fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9617 const o = self.ng.object;
9618 const pt = self.ng.pt;
9619 const zcu = pt.zcu;
9620 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9621 const dest_ptr = try self.resolveInst(bin_op.lhs);
9622 const ptr_ty = self.typeOf(bin_op.lhs);
9623 const operand_ty = ptr_ty.childType(zcu);
9624
9625 const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false;
9626 if (val_is_undef) {
9627 const owner_mod = self.ng.ownerModule();
9628
9629 // Even if safety is disabled, we still emit a memset to undefined since it conveys
9630 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
9631 // between using 0xaa or actual undefined for the fill byte.
9632 //
9633 // However, for Debug builds specifically, we avoid emitting the memset because LLVM
9634 // will neither use the information nor get rid of the memset, thus leaving an
9635 // unexpected call in the user's code. This is problematic if the code in question is
9636 // not ready to correctly make calls yet, such as in our early PIE startup code, or in
9637 // the early stages of a dynamic linker, etc.
9638 if (!safety and owner_mod.optimize_mode == .Debug) {
9639 return .none;
9640 }
9641
9642 const ptr_info = ptr_ty.ptrInfo(zcu);
9643 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
9644 if (needs_bitmask) {
9645 // TODO: only some bits are to be undef, we cannot write with a simple memset.
9646 // meanwhile, ignore the write rather than stomping over valid bits.
9647 // https://github.com/ziglang/zig/issues/15337
9648 return .none;
9649 }
9650
9651 self.maybeMarkAllowZeroAccess(ptr_info);
9652
9653 const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu));
9654 _ = try self.wip.callMemSet(
9655 dest_ptr,
9656 ptr_ty.ptrAlignment(zcu).toLlvm(),
9657 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
9658 len,
9659 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9660 self.disable_intrinsics,
9661 );
9662 if (safety and owner_mod.valgrind) {
9663 try self.valgrindMarkUndef(dest_ptr, len);
9664 }
9665 return .none;
9666 }
9667
9668 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9669
9670 const src_operand = try self.resolveInst(bin_op.rhs);
9671 try self.store(dest_ptr, ptr_ty, src_operand, .none);
9672 return .none;
9673 }
9674
9675 fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9676 const pt = fg.ng.pt;
9677 const zcu = pt.zcu;
9678 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9679 const ptr_ty = fg.typeOf(ty_op.operand);
9680 const ptr_info = ptr_ty.ptrInfo(zcu);
9681 const ptr = try fg.resolveInst(ty_op.operand);
9682 fg.maybeMarkAllowZeroAccess(ptr_info);
9683 return fg.load(ptr, ptr_ty);
9684 }
9685
9686 fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void {
9687 _ = inst;
9688 _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
9689 _ = try self.wip.@"unreachable"();
9690 }
9691
9692 fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9693 _ = inst;
9694 _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
9695 return .none;
9696 }
9697
9698 fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9699 _ = inst;
9700 const o = self.ng.object;
9701 const pt = self.ng.pt;
9702 const llvm_usize = try o.lowerType(pt, Type.usize);
9703 if (!target_util.supportsReturnAddress(self.ng.pt.zcu.getTarget(), self.ng.ownerModule().optimize_mode)) {
9704 // https://github.com/ziglang/zig/issues/11946
9705 return o.builder.intValue(llvm_usize, 0);
9706 }
9707 const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
9708 return self.wip.cast(.ptrtoint, result, llvm_usize, "");
9709 }
9710
9711 fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9712 _ = inst;
9713 const o = self.ng.object;
9714 const pt = self.ng.pt;
9715 const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
9716 return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), "");
9717 }
9718
9719 fn airCmpxchg(
9720 self: *FuncGen,
9721 inst: Air.Inst.Index,
9722 kind: Builder.Function.Instruction.CmpXchg.Kind,
9723 ) !Builder.Value {
9724 const o = self.ng.object;
9725 const pt = self.ng.pt;
9726 const zcu = pt.zcu;
9727 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
9728 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
9729 const ptr = try self.resolveInst(extra.ptr);
9730 const ptr_ty = self.typeOf(extra.ptr);
9731 var expected_value = try self.resolveInst(extra.expected_value);
9732 var new_value = try self.resolveInst(extra.new_value);
9733 const operand_ty = ptr_ty.childType(zcu);
9734 const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9735 const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9736 if (llvm_abi_ty != .none) {
9737 // operand needs widening and truncating
9738 const signedness: Builder.Function.Instruction.Cast.Signedness =
9739 if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
9740 expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
9741 new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
9742 }
9743
9744 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9745
9746 const result = try self.wip.cmpxchg(
9747 kind,
9748 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9749 ptr,
9750 expected_value,
9751 new_value,
9752 self.sync_scope,
9753 toLlvmAtomicOrdering(extra.successOrder()),
9754 toLlvmAtomicOrdering(extra.failureOrder()),
9755 ptr_ty.ptrAlignment(zcu).toLlvm(),
9756 "",
9757 );
9758
9759 const optional_ty = self.typeOfIndex(inst);
9760
9761 var payload = try self.wip.extractValue(result, &.{0}, "");
9762 if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
9763 const success_bit = try self.wip.extractValue(result, &.{1}, "");
9764
9765 if (optional_ty.optionalReprIsPayload(zcu)) {
9766 const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
9767 return self.wip.select(.normal, success_bit, zero, payload, "");
9768 }
9769
9770 comptime assert(optional_layout_version == 3);
9771
9772 const non_null_bit = try self.wip.not(success_bit, "");
9773 return buildOptional(self, optional_ty, payload, non_null_bit);
9774 }
9775
9776 fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9777 const o = self.ng.object;
9778 const pt = self.ng.pt;
9779 const zcu = pt.zcu;
9780 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
9781 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
9782 const ptr = try self.resolveInst(pl_op.operand);
9783 const ptr_ty = self.typeOf(pl_op.operand);
9784 const operand_ty = ptr_ty.childType(zcu);
9785 const operand = try self.resolveInst(extra.operand);
9786 const is_signed_int = operand_ty.isSignedInt(zcu);
9787 const is_float = operand_ty.isRuntimeFloat();
9788 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
9789 const ordering = toLlvmAtomicOrdering(extra.ordering());
9790 const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, op == .xchg);
9791 const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9792
9793 const access_kind: Builder.MemoryAccessKind =
9794 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9795 const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9796
9797 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9798
9799 if (llvm_abi_ty != .none) {
9800 // operand needs widening and truncating or bitcasting.
9801 return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
9802 access_kind,
9803 op,
9804 ptr,
9805 try self.wip.cast(
9806 if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
9807 operand,
9808 llvm_abi_ty,
9809 "",
9810 ),
9811 self.sync_scope,
9812 ordering,
9813 ptr_alignment,
9814 "",
9815 ), llvm_operand_ty, "");
9816 }
9817
9818 if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw(
9819 access_kind,
9820 op,
9821 ptr,
9822 operand,
9823 self.sync_scope,
9824 ordering,
9825 ptr_alignment,
9826 "",
9827 );
9828
9829 // It's a pointer but we need to treat it as an int.
9830 return self.wip.cast(.inttoptr, try self.wip.atomicrmw(
9831 access_kind,
9832 op,
9833 ptr,
9834 try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""),
9835 self.sync_scope,
9836 ordering,
9837 ptr_alignment,
9838 "",
9839 ), llvm_operand_ty, "");
9840 }
9841
9842 fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9843 const o = self.ng.object;
9844 const pt = self.ng.pt;
9845 const zcu = pt.zcu;
9846 const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
9847 const ptr = try self.resolveInst(atomic_load.ptr);
9848 const ptr_ty = self.typeOf(atomic_load.ptr);
9849 const info = ptr_ty.ptrInfo(zcu);
9850 const elem_ty = Type.fromInterned(info.child);
9851 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
9852 const ordering = toLlvmAtomicOrdering(atomic_load.order);
9853 const llvm_abi_ty = try o.getAtomicAbiType(pt, elem_ty, false);
9854 const ptr_alignment = (if (info.flags.alignment != .none)
9855 @as(InternPool.Alignment, info.flags.alignment)
9856 else
9857 Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
9858 const access_kind: Builder.MemoryAccessKind =
9859 if (info.flags.is_volatile) .@"volatile" else .normal;
9860 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9861
9862 self.maybeMarkAllowZeroAccess(info);
9863
9864 if (llvm_abi_ty != .none) {
9865 // operand needs widening and truncating
9866 const loaded = try self.wip.loadAtomic(
9867 access_kind,
9868 llvm_abi_ty,
9869 ptr,
9870 self.sync_scope,
9871 ordering,
9872 ptr_alignment,
9873 "",
9874 );
9875 return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
9876 }
9877 return self.wip.loadAtomic(
9878 access_kind,
9879 elem_llvm_ty,
9880 ptr,
9881 self.sync_scope,
9882 ordering,
9883 ptr_alignment,
9884 "",
9885 );
9886 }
9887
9888 fn airAtomicStore(
9889 self: *FuncGen,
9890 inst: Air.Inst.Index,
9891 ordering: Builder.AtomicOrdering,
9892 ) !Builder.Value {
9893 const o = self.ng.object;
9894 const pt = self.ng.pt;
9895 const zcu = pt.zcu;
9896 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9897 const ptr_ty = self.typeOf(bin_op.lhs);
9898 const operand_ty = ptr_ty.childType(zcu);
9899 if (!operand_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) return .none;
9900 const ptr = try self.resolveInst(bin_op.lhs);
9901 var element = try self.resolveInst(bin_op.rhs);
9902 const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9903
9904 if (llvm_abi_ty != .none) {
9905 // operand needs widening
9906 element = try self.wip.conv(
9907 if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
9908 element,
9909 llvm_abi_ty,
9910 "",
9911 );
9912 }
9913
9914 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9915
9916 try self.store(ptr, ptr_ty, element, ordering);
9917 return .none;
9918 }
9919
9920 fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9921 const o = self.ng.object;
9922 const pt = self.ng.pt;
9923 const zcu = pt.zcu;
9924 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9925 const dest_slice = try self.resolveInst(bin_op.lhs);
9926 const ptr_ty = self.typeOf(bin_op.lhs);
9927 const elem_ty = self.typeOf(bin_op.rhs);
9928 const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
9929 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
9930 const access_kind: Builder.MemoryAccessKind =
9931 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9932
9933 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9934
9935 if (try self.air.value(bin_op.rhs, pt)) |elem_val| {
9936 if (elem_val.isUndef(zcu)) {
9937 // Even if safety is disabled, we still emit a memset to undefined since it conveys
9938 // extra information to LLVM. However, safety makes the difference between using
9939 // 0xaa or actual undefined for the fill byte.
9940 const fill_byte = if (safety)
9941 try o.builder.intValue(.i8, 0xaa)
9942 else
9943 try o.builder.undefValue(.i8);
9944 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9945 _ = try self.wip.callMemSet(
9946 dest_ptr,
9947 dest_ptr_align,
9948 fill_byte,
9949 len,
9950 access_kind,
9951 self.disable_intrinsics,
9952 );
9953 const owner_mod = self.ng.ownerModule();
9954 if (safety and owner_mod.valgrind) {
9955 try self.valgrindMarkUndef(dest_ptr, len);
9956 }
9957 return .none;
9958 }
9959
9960 // Test if the element value is compile-time known to be a
9961 // repeating byte pattern, for example, `@as(u64, 0)` has a
9962 // repeating byte pattern of 0 bytes. In such case, the memset
9963 // intrinsic can be used.
9964 if (try elem_val.hasRepeatedByteRepr(pt)) |byte_val| {
9965 const fill_byte = try o.builder.intValue(.i8, byte_val);
9966 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9967 _ = try self.wip.callMemSet(
9968 dest_ptr,
9969 dest_ptr_align,
9970 fill_byte,
9971 len,
9972 access_kind,
9973 self.disable_intrinsics,
9974 );
9975 return .none;
9976 }
9977 }
9978
9979 const value = try self.resolveInst(bin_op.rhs);
9980 const elem_abi_size = elem_ty.abiSize(zcu);
9981
9982 if (elem_abi_size == 1) {
9983 // In this case we can take advantage of LLVM's intrinsic.
9984 const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
9985 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9986
9987 _ = try self.wip.callMemSet(
9988 dest_ptr,
9989 dest_ptr_align,
9990 fill_byte,
9991 len,
9992 access_kind,
9993 self.disable_intrinsics,
9994 );
9995 return .none;
9996 }
9997
9998 // non-byte-sized element. lower with a loop. something like this:
9999
10000 // entry:
10001 // ...
10002 // %end_ptr = getelementptr %ptr, %len
10003 // br %loop
10004 // loop:
10005 // %it_ptr = phi body %next_ptr, entry %ptr
10006 // %end = cmp eq %it_ptr, %end_ptr
10007 // br %end, %body, %end
10008 // body:
10009 // store %it_ptr, %value
10010 // %next_ptr = getelementptr %it_ptr, 1
10011 // br %loop
10012 // end:
10013 // ...
10014 const entry_block = self.wip.cursor.block;
10015 const loop_block = try self.wip.block(2, "InlineMemsetLoop");
10016 const body_block = try self.wip.block(1, "InlineMemsetBody");
10017 const end_block = try self.wip.block(1, "InlineMemsetEnd");
10018
10019 const llvm_usize_ty = try o.lowerType(pt, Type.usize);
10020 const len = switch (ptr_ty.ptrSize(zcu)) {
10021 .slice => try self.wip.extractValue(dest_slice, &.{1}, ""),
10022 .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),
10023 .many, .c => unreachable,
10024 };
10025 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
10026 const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
10027 _ = try self.wip.br(loop_block);
10028
10029 self.wip.cursor = .{ .block = loop_block };
10030 const it_ptr = try self.wip.phi(.ptr, "");
10031 const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
10032 _ = try self.wip.brCond(end, body_block, end_block, .none);
10033
10034 self.wip.cursor = .{ .block = body_block };
10035 const elem_abi_align = elem_ty.abiAlignment(zcu);
10036 const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
10037 if (isByRef(elem_ty, zcu)) {
10038 _ = try self.wip.callMemCpy(
10039 it_ptr.toValue(),
10040 it_ptr_align,
10041 value,
10042 elem_abi_align.toLlvm(),
10043 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
10044 access_kind,
10045 self.disable_intrinsics,
10046 );
10047 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
10048 const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
10049 try o.builder.intValue(llvm_usize_ty, 1),
10050 }, "");
10051 _ = try self.wip.br(loop_block);
10052
10053 self.wip.cursor = .{ .block = end_block };
10054 it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
10055 return .none;
10056 }
10057
10058 fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10059 const pt = self.ng.pt;
10060 const zcu = pt.zcu;
10061 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
10062 const dest_slice = try self.resolveInst(bin_op.lhs);
10063 const dest_ptr_ty = self.typeOf(bin_op.lhs);
10064 const src_slice = try self.resolveInst(bin_op.rhs);
10065 const src_ptr_ty = self.typeOf(bin_op.rhs);
10066 const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
10067 const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
10068 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
10069 const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
10070 dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
10071
10072 self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
10073 self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
10074
10075 _ = try self.wip.callMemCpy(
10076 dest_ptr,
10077 dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
10078 src_ptr,
10079 src_ptr_ty.ptrAlignment(zcu).toLlvm(),
10080 len,
10081 access_kind,
10082 self.disable_intrinsics,
10083 );
10084 return .none;
10085 }
10086
10087 fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10088 const pt = self.ng.pt;
10089 const zcu = pt.zcu;
10090 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
10091 const dest_slice = try self.resolveInst(bin_op.lhs);
10092 const dest_ptr_ty = self.typeOf(bin_op.lhs);
10093 const src_slice = try self.resolveInst(bin_op.rhs);
10094 const src_ptr_ty = self.typeOf(bin_op.rhs);
10095 const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
10096 const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
10097 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
10098 const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
10099 dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
10100
10101 _ = try self.wip.callMemMove(
10102 dest_ptr,
10103 dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
10104 src_ptr,
10105 src_ptr_ty.ptrAlignment(zcu).toLlvm(),
10106 len,
10107 access_kind,
10108 );
10109 return .none;
10110 }
10111
10112 fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10113 const o = self.ng.object;
10114 const pt = self.ng.pt;
10115 const zcu = pt.zcu;
10116 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
10117 const un_ptr_ty = self.typeOf(bin_op.lhs);
10118 const un_ty = un_ptr_ty.childType(zcu);
10119 const layout = un_ty.unionGetLayout(zcu);
10120 if (layout.tag_size == 0) return .none;
10121
10122 const access_kind: Builder.MemoryAccessKind =
10123 if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
10124
10125 self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
10126
10127 const union_ptr = try self.resolveInst(bin_op.lhs);
10128 const new_tag = try self.resolveInst(bin_op.rhs);
10129 if (layout.payload_size == 0) {
10130 // TODO alignment on this store
10131 _ = try self.wip.store(access_kind, new_tag, union_ptr, .default);
10132 return .none;
10133 }
10134 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10135 const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
10136 // TODO alignment on this store
10137 _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, .default);
10138 return .none;
10139 }
10140
10141 fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10142 const o = self.ng.object;
10143 const pt = self.ng.pt;
10144 const zcu = pt.zcu;
10145 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10146 const un_ty = self.typeOf(ty_op.operand);
10147 const layout = un_ty.unionGetLayout(zcu);
10148 if (layout.tag_size == 0) return .none;
10149 const union_handle = try self.resolveInst(ty_op.operand);
10150 if (isByRef(un_ty, zcu)) {
10151 const llvm_un_ty = try o.lowerType(pt, un_ty);
10152 if (layout.payload_size == 0)
10153 return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");
10154 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10155 const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");
10156 const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
10157 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
10158 } else {
10159 if (layout.payload_size == 0) return union_handle;
10160 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10161 return self.wip.extractValue(union_handle, &.{tag_index}, "");
10162 }
10163 }
10164
10165 fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value {
10166 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10167 const operand = try self.resolveInst(un_op);
10168 const operand_ty = self.typeOf(un_op);
10169
10170 return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
10171 }
10172
10173 fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10174 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10175 const operand = try self.resolveInst(un_op);
10176 const operand_ty = self.typeOf(un_op);
10177
10178 return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
10179 }
10180
10181 fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10182 const o = self.ng.object;
10183 const pt = self.ng.pt;
10184 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10185 const inst_ty = self.typeOfIndex(inst);
10186 const operand_ty = self.typeOf(ty_op.operand);
10187 const operand = try self.resolveInst(ty_op.operand);
10188
10189 const result = try self.wip.callIntrinsic(
10190 .normal,
10191 .none,
10192 intrinsic,
10193 &.{try o.lowerType(pt, operand_ty)},
10194 &.{ operand, .false },
10195 "",
10196 );
10197 return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10198 }
10199
10200 fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10201 const o = self.ng.object;
10202 const pt = self.ng.pt;
10203 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10204 const inst_ty = self.typeOfIndex(inst);
10205 const operand_ty = self.typeOf(ty_op.operand);
10206 const operand = try self.resolveInst(ty_op.operand);
10207
10208 const result = try self.wip.callIntrinsic(
10209 .normal,
10210 .none,
10211 intrinsic,
10212 &.{try o.lowerType(pt, operand_ty)},
10213 &.{operand},
10214 "",
10215 );
10216 return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10217 }
10218
10219 fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10220 const o = self.ng.object;
10221 const pt = self.ng.pt;
10222 const zcu = pt.zcu;
10223 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10224 const operand_ty = self.typeOf(ty_op.operand);
10225 var bits = operand_ty.intInfo(zcu).bits;
10226 assert(bits % 8 == 0);
10227
10228 const inst_ty = self.typeOfIndex(inst);
10229 var operand = try self.resolveInst(ty_op.operand);
10230 var llvm_operand_ty = try o.lowerType(pt, operand_ty);
10231
10232 if (bits % 16 == 8) {
10233 // If not an even byte-multiple, we need zero-extend + shift-left 1 byte
10234 // The truncated result at the end will be the correct bswap
10235 const scalar_ty = try o.builder.intType(@intCast(bits + 8));
10236 if (operand_ty.zigTypeTag(zcu) == .vector) {
10237 const vec_len = operand_ty.vectorLen(zcu);
10238 llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
10239 } else llvm_operand_ty = scalar_ty;
10240
10241 const shift_amt =
10242 try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
10243 const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
10244 operand = try self.wip.bin(.shl, extended, shift_amt, "");
10245
10246 bits = bits + 8;
10247 }
10248
10249 const result =
10250 try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
10251 return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10252 }
10253
10254 fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10255 const o = self.ng.object;
10256 const pt = self.ng.pt;
10257 const zcu = pt.zcu;
10258 const ip = &zcu.intern_pool;
10259 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10260 const operand = try self.resolveInst(ty_op.operand);
10261 const error_set_ty = ty_op.ty.toType();
10262
10263 const names = error_set_ty.errorSetNames(zcu);
10264 const valid_block = try self.wip.block(@intCast(names.len), "Valid");
10265 const invalid_block = try self.wip.block(1, "Invalid");
10266 const end_block = try self.wip.block(2, "End");
10267 var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
10268 defer wip_switch.finish(&self.wip);
10269
10270 for (0..names.len) |name_index| {
10271 const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
10272 const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int);
10273 try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
10274 }
10275 self.wip.cursor = .{ .block = valid_block };
10276 _ = try self.wip.br(end_block);
10277
10278 self.wip.cursor = .{ .block = invalid_block };
10279 _ = try self.wip.br(end_block);
10280
10281 self.wip.cursor = .{ .block = end_block };
10282 const phi = try self.wip.phi(.i1, "");
10283 phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
10284 return phi.toValue();
10285 }
10286
10287 fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10288 const o = self.ng.object;
10289 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10290 const operand = try self.resolveInst(un_op);
10291 const enum_ty = self.typeOf(un_op);
10292
10293 const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty);
10294 return self.wip.call(
10295 .normal,
10296 .fastcc,
10297 .none,
10298 llvm_fn.typeOf(&o.builder),
10299 llvm_fn.toValue(&o.builder),
10300 &.{operand},
10301 "",
10302 );
10303 }
10304
10305 fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index {
10306 const o = self.ng.object;
10307 const pt = self.ng.pt;
10308 const zcu = pt.zcu;
10309 const ip = &zcu.intern_pool;
10310 const enum_type = ip.loadEnumType(enum_ty.toIntern());
10311
10312 // TODO: detect when the type changes and re-emit this function.
10313 const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
10314 if (gop.found_existing) return gop.value_ptr.*;
10315 errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
10316
10317 const target = &zcu.root_mod.resolved_target.result;
10318 const function_index = try o.builder.addFunction(
10319 try o.builder.fnType(.i1, &.{try o.lowerType(pt, Type.fromInterned(enum_type.tag_ty))}, .normal),
10320 try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_type.name.fmt(ip)}),
10321 toLlvmAddressSpace(.generic, target),
10322 );
10323
10324 var attributes: Builder.FunctionAttributes.Wip = .{};
10325 defer attributes.deinit(&o.builder);
10326 try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
10327
10328 function_index.setLinkage(.internal, &o.builder);
10329 function_index.setCallConv(.fastcc, &o.builder);
10330 function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
10331 gop.value_ptr.* = function_index;
10332
10333 var wip = try Builder.WipFunction.init(&o.builder, .{
10334 .function = function_index,
10335 .strip = true,
10336 });
10337 defer wip.deinit();
10338 wip.cursor = .{ .block = try wip.block(0, "Entry") };
10339
10340 const named_block = try wip.block(@intCast(enum_type.names.len), "Named");
10341 const unnamed_block = try wip.block(1, "Unnamed");
10342 const tag_int_value = wip.arg(0);
10343 var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len), .none);
10344 defer wip_switch.finish(&wip);
10345
10346 for (0..enum_type.names.len) |field_index| {
10347 const this_tag_int_value = try o.lowerValue(
10348 pt,
10349 (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(),
10350 );
10351 try wip_switch.addCase(this_tag_int_value, named_block, &wip);
10352 }
10353 wip.cursor = .{ .block = named_block };
10354 _ = try wip.ret(.true);
10355
10356 wip.cursor = .{ .block = unnamed_block };
10357 _ = try wip.ret(.false);
10358
10359 try wip.finish();
10360 return function_index;
10361 }
10362
10363 fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10364 const o = self.ng.object;
10365 const pt = self.ng.pt;
10366 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10367 const operand = try self.resolveInst(un_op);
10368 const enum_ty = self.typeOf(un_op);
10369
10370 const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty);
10371 return self.wip.call(
10372 .normal,
10373 .fastcc,
10374 .none,
10375 llvm_fn.typeOf(&o.builder),
10376 llvm_fn.toValue(&o.builder),
10377 &.{operand},
10378 "",
10379 );
10380 }
10381
10382 fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10383 const o = self.ng.object;
10384 const pt = self.ng.pt;
10385 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10386 const operand = try self.resolveInst(un_op);
10387 const slice_ty = self.typeOfIndex(inst);
10388 const slice_llvm_ty = try o.lowerType(pt, slice_ty);
10389
10390 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
10391 const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
10392
10393 const error_name_table_ptr = try self.getErrorNameTable();
10394 const error_name_table =
10395 try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
10396 const error_name_ptr =
10397 try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
10398 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
10399 }
10400
10401 fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10402 const o = self.ng.object;
10403 const pt = self.ng.pt;
10404 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10405 const scalar = try self.resolveInst(ty_op.operand);
10406 const vector_ty = self.typeOfIndex(inst);
10407 return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, "");
10408 }
10409
10410 fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10411 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10412 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
10413 const pred = try self.resolveInst(pl_op.operand);
10414 const a = try self.resolveInst(extra.lhs);
10415 const b = try self.resolveInst(extra.rhs);
10416
10417 return self.wip.select(.normal, pred, a, b, "");
10418 }
10419
10420 fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10421 const o = fg.ng.object;
10422 const pt = fg.ng.pt;
10423 const zcu = pt.zcu;
10424 const gpa = zcu.gpa;
10425
10426 const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
10427
10428 const operand = try fg.resolveInst(unwrapped.operand);
10429 const mask = unwrapped.mask;
10430 const operand_ty = fg.typeOf(unwrapped.operand);
10431 const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10432 const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty);
10433 const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10434 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
10435 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10436 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10437
10438 // LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
10439 // And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
10440 // least a bit". So, there are two cases here.
10441 //
10442 // If the operand length equals the mask length, we do just the one `shufflevector`, where
10443 // the second operand is a constant vector with comptime-known elements at the right indices
10444 // and poison values elsewhere (in the indices which won't be selected).
10445 //
10446 // Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime
10447 // operand with an all-poison vector to extract and correctly position all of the runtime
10448 // elements. We also make a constant vector with all of the comptime elements correctly
10449 // positioned. Then, our second instruction selects elements from those "runtime-or-poison"
10450 // and "comptime-or-poison" vectors to compute the result.
10451
10452 // This buffer is used primarily for the mask constants.
10453 const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
10454 defer gpa.free(llvm_elem_buf);
10455
10456 // ...but first, we'll collect all of the comptime-known values.
10457 var any_defined_comptime_value = false;
10458 for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
10459 llvm_elem.* = switch (mask_elem.unwrap()) {
10460 .elem => llvm_poison_elem,
10461 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
10462 any_defined_comptime_value = true;
10463 break :elem try o.lowerValue(pt, val);
10464 } else llvm_poison_elem,
10465 };
10466 }
10467 // This vector is like the result, but runtime elements are replaced with poison.
10468 const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
10469 break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
10470 } else try o.builder.poisonValue(llvm_result_ty);
10471
10472 if (operand_ty.vectorLen(zcu) == mask.len) {
10473 // input length equals mask/output length, so we lower to one instruction
10474 for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
10475 llvm_elem.* = switch (mask_elem.unwrap()) {
10476 .elem => |idx| try o.builder.intConst(.i32, idx),
10477 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10478 break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10479 } else llvm_poison_mask_elem,
10480 };
10481 }
10482 return fg.wip.shuffleVector(
10483 operand,
10484 comptime_and_poison,
10485 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10486 "",
10487 );
10488 }
10489
10490 for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
10491 llvm_elem.* = switch (mask_elem.unwrap()) {
10492 .elem => |idx| try o.builder.intConst(.i32, idx),
10493 .value => llvm_poison_mask_elem,
10494 };
10495 }
10496 // This vector is like our result, but all comptime-known elements are poison.
10497 const runtime_and_poison = try fg.wip.shuffleVector(
10498 operand,
10499 try o.builder.poisonValue(llvm_operand_ty),
10500 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10501 "",
10502 );
10503
10504 if (!any_defined_comptime_value) {
10505 // `comptime_and_poison` is just poison; a second shuffle would be a nop.
10506 return runtime_and_poison;
10507 }
10508
10509 // In this second shuffle, the inputs, the mask, and the output all have the same length.
10510 for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
10511 llvm_elem.* = switch (mask_elem.unwrap()) {
10512 .elem => try o.builder.intConst(.i32, elem_idx),
10513 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10514 break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10515 } else llvm_poison_mask_elem,
10516 };
10517 }
10518 // Merge the runtime and comptime elements with the mask we just built.
10519 return fg.wip.shuffleVector(
10520 runtime_and_poison,
10521 comptime_and_poison,
10522 try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10523 "",
10524 );
10525 }
10526
10527 fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10528 const o = fg.ng.object;
10529 const pt = fg.ng.pt;
10530 const zcu = pt.zcu;
10531 const gpa = zcu.gpa;
10532
10533 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
10534
10535 const mask = unwrapped.mask;
10536 const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10537 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10538 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10539
10540 // This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
10541 // length of the longer one with an initial `shufflevector` if necessary, and then do the
10542 // actual computation with a second `shufflevector`.
10543
10544 const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
10545 const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
10546 const operand_len: u32 = @max(operand_a_len, operand_b_len);
10547
10548 // If we need to extend an operand, this is the type that mask will have.
10549 const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
10550
10551 const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
10552 defer gpa.free(llvm_elem_buf);
10553
10554 const operand_a: Builder.Value = extend: {
10555 const raw = try fg.resolveInst(unwrapped.operand_a);
10556 if (operand_a_len == operand_len) break :extend raw;
10557 // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10558 const mask_elems = llvm_elem_buf[0..operand_len];
10559 for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| {
10560 llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10561 }
10562 @memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
10563 const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
10564 break :extend try fg.wip.shuffleVector(
10565 raw,
10566 try o.builder.poisonValue(llvm_this_operand_ty),
10567 try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10568 "",
10569 );
10570 };
10571 const operand_b: Builder.Value = extend: {
10572 const raw = try fg.resolveInst(unwrapped.operand_b);
10573 if (operand_b_len == operand_len) break :extend raw;
10574 // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10575 const mask_elems = llvm_elem_buf[0..operand_len];
10576 for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| {
10577 llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10578 }
10579 @memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
10580 const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
10581 break :extend try fg.wip.shuffleVector(
10582 raw,
10583 try o.builder.poisonValue(llvm_this_operand_ty),
10584 try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10585 "",
10586 );
10587 };
10588
10589 // `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
10590 // an initial shuffle if necessary). Now for the easy bit.
10591
10592 const mask_elems = llvm_elem_buf[0..mask.len];
10593 for (mask, mask_elems) |mask_elem, *llvm_mask_elem| {
10594 llvm_mask_elem.* = switch (mask_elem.unwrap()) {
10595 .a_elem => |idx| try o.builder.intConst(.i32, idx),
10596 .b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx),
10597 .undef => llvm_poison_mask_elem,
10598 };
10599 }
10600 return fg.wip.shuffleVector(
10601 operand_a,
10602 operand_b,
10603 try o.builder.vectorValue(llvm_mask_ty, mask_elems),
10604 "",
10605 );
10606 }
10607
10608 /// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
10609 ///
10610 /// Equivalent to:
10611 /// reduce: {
10612 /// var i: usize = 0;
10613 /// var accum: T = init;
10614 /// while (i < vec.len) : (i += 1) {
10615 /// accum = llvm_fn(accum, vec[i]);
10616 /// }
10617 /// break :reduce accum;
10618 /// }
10619 ///
10620 fn buildReducedCall(
10621 self: *FuncGen,
10622 llvm_fn: Builder.Function.Index,
10623 operand_vector: Builder.Value,
10624 vector_len: usize,
10625 accum_init: Builder.Value,
10626 ) !Builder.Value {
10627 const o = self.ng.object;
10628 const pt = self.ng.pt;
10629 const usize_ty = try o.lowerType(pt, Type.usize);
10630 const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
10631 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
10632
10633 // Allocate and initialize our mutable variables
10634 const i_ptr = try self.buildAlloca(usize_ty, .default);
10635 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
10636 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
10637 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
10638
10639 // Setup the loop
10640 const loop = try self.wip.block(2, "ReduceLoop");
10641 const loop_exit = try self.wip.block(1, "AfterReduce");
10642 _ = try self.wip.br(loop);
10643 {
10644 self.wip.cursor = .{ .block = loop };
10645
10646 // while (i < vec.len)
10647 const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
10648 const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
10649 const loop_then = try self.wip.block(1, "ReduceLoopThen");
10650
10651 _ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
10652
10653 {
10654 self.wip.cursor = .{ .block = loop_then };
10655
10656 // accum = f(accum, vec[i]);
10657 const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10658 const element = try self.wip.extractElement(operand_vector, i, "");
10659 const new_accum = try self.wip.call(
10660 .normal,
10661 .ccc,
10662 .none,
10663 llvm_fn.typeOf(&o.builder),
10664 llvm_fn.toValue(&o.builder),
10665 &.{ accum, element },
10666 "",
10667 );
10668 _ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
10669
10670 // i += 1
10671 const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
10672 _ = try self.wip.store(.normal, new_i, i_ptr, .default);
10673 _ = try self.wip.br(loop);
10674 }
10675 }
10676
10677 self.wip.cursor = .{ .block = loop_exit };
10678 return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10679 }
10680
10681 fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10682 const o = self.ng.object;
10683 const pt = self.ng.pt;
10684 const zcu = pt.zcu;
10685 const target = zcu.getTarget();
10686
10687 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
10688 const operand = try self.resolveInst(reduce.operand);
10689 const operand_ty = self.typeOf(reduce.operand);
10690 const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10691 const scalar_ty = self.typeOfIndex(inst);
10692 const llvm_scalar_ty = try o.lowerType(pt, scalar_ty);
10693
10694 switch (reduce.operation) {
10695 .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10696 .And => .@"vector.reduce.and",
10697 .Or => .@"vector.reduce.or",
10698 .Xor => .@"vector.reduce.xor",
10699 else => unreachable,
10700 }, &.{llvm_operand_ty}, &.{operand}, ""),
10701 .Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
10702 .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10703 .Min => if (scalar_ty.isSignedInt(zcu))
10704 .@"vector.reduce.smin"
10705 else
10706 .@"vector.reduce.umin",
10707 .Max => if (scalar_ty.isSignedInt(zcu))
10708 .@"vector.reduce.smax"
10709 else
10710 .@"vector.reduce.umax",
10711 else => unreachable,
10712 }, &.{llvm_operand_ty}, &.{operand}, ""),
10713 .float => if (intrinsicsAllowed(scalar_ty, target))
10714 return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10715 .Min => .@"vector.reduce.fmin",
10716 .Max => .@"vector.reduce.fmax",
10717 else => unreachable,
10718 }, &.{llvm_operand_ty}, &.{operand}, ""),
10719 else => unreachable,
10720 },
10721 .Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
10722 .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10723 .Add => .@"vector.reduce.add",
10724 .Mul => .@"vector.reduce.mul",
10725 else => unreachable,
10726 }, &.{llvm_operand_ty}, &.{operand}, ""),
10727 .float => if (intrinsicsAllowed(scalar_ty, target))
10728 return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10729 .Add => .@"vector.reduce.fadd",
10730 .Mul => .@"vector.reduce.fmul",
10731 else => unreachable,
10732 }, &.{llvm_operand_ty}, &.{ switch (reduce.operation) {
10733 .Add => try o.builder.fpValue(llvm_scalar_ty, -0.0),
10734 .Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0),
10735 else => unreachable,
10736 }, operand }, ""),
10737 else => unreachable,
10738 },
10739 }
10740
10741 // Reduction could not be performed with intrinsics.
10742 // Use a manual loop over a softfloat call instead.
10743 const float_bits = scalar_ty.floatBits(target);
10744 const fn_name = switch (reduce.operation) {
10745 .Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
10746 libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10747 }),
10748 .Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
10749 libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10750 }),
10751 .Add => try o.builder.strtabStringFmt("__add{s}f3", .{
10752 compilerRtFloatAbbrev(float_bits),
10753 }),
10754 .Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
10755 compilerRtFloatAbbrev(float_bits),
10756 }),
10757 else => unreachable,
10758 };
10759
10760 const libc_fn =
10761 try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty);
10762 const init_val = switch (llvm_scalar_ty) {
10763 .i16 => try o.builder.intValue(.i16, @as(i16, @bitCast(
10764 @as(f16, switch (reduce.operation) {
10765 .Min, .Max => std.math.nan(f16),
10766 .Add => -0.0,
10767 .Mul => 1.0,
10768 else => unreachable,
10769 }),
10770 ))),
10771 .i80 => try o.builder.intValue(.i80, @as(i80, @bitCast(
10772 @as(f80, switch (reduce.operation) {
10773 .Min, .Max => std.math.nan(f80),
10774 .Add => -0.0,
10775 .Mul => 1.0,
10776 else => unreachable,
10777 }),
10778 ))),
10779 .i128 => try o.builder.intValue(.i128, @as(i128, @bitCast(
10780 @as(f128, switch (reduce.operation) {
10781 .Min, .Max => std.math.nan(f128),
10782 .Add => -0.0,
10783 .Mul => 1.0,
10784 else => unreachable,
10785 }),
10786 ))),
10787 else => unreachable,
10788 };
10789 return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val);
10790 }
10791
10792 fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10793 const o = self.ng.object;
10794 const pt = self.ng.pt;
10795 const zcu = pt.zcu;
10796 const ip = &zcu.intern_pool;
10797 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10798 const result_ty = self.typeOfIndex(inst);
10799 const len: usize = @intCast(result_ty.arrayLen(zcu));
10800 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
10801 const llvm_result_ty = try o.lowerType(pt, result_ty);
10802
10803 switch (result_ty.zigTypeTag(zcu)) {
10804 .vector => {
10805 var vector = try o.builder.poisonValue(llvm_result_ty);
10806 for (elements, 0..) |elem, i| {
10807 const index_u32 = try o.builder.intValue(.i32, i);
10808 const llvm_elem = try self.resolveInst(elem);
10809 vector = try self.wip.insertElement(vector, llvm_elem, index_u32, "");
10810 }
10811 return vector;
10812 },
10813 .@"struct" => {
10814 if (zcu.typeToPackedStruct(result_ty)) |struct_type| {
10815 const backing_int_ty = struct_type.backingIntTypeUnordered(ip);
10816 assert(backing_int_ty != .none);
10817 const big_bits = Type.fromInterned(backing_int_ty).bitSize(zcu);
10818 const int_ty = try o.builder.intType(@intCast(big_bits));
10819 comptime assert(Type.packed_struct_layout_version == 2);
10820 var running_int = try o.builder.intValue(int_ty, 0);
10821 var running_bits: u16 = 0;
10822 for (elements, struct_type.field_types.get(ip)) |elem, field_ty| {
10823 if (!Type.fromInterned(field_ty).hasRuntimeBitsIgnoreComptime(zcu)) continue;
10824
10825 const non_int_val = try self.resolveInst(elem);
10826 const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
10827 const small_int_ty = try o.builder.intType(ty_bit_size);
10828 const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
10829 try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
10830 else
10831 try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
10832 const shift_rhs = try o.builder.intValue(int_ty, running_bits);
10833 const extended_int_val =
10834 try self.wip.conv(.unsigned, small_int_val, int_ty, "");
10835 const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, "");
10836 running_int = try self.wip.bin(.@"or", running_int, shifted, "");
10837 running_bits += ty_bit_size;
10838 }
10839 return running_int;
10840 }
10841
10842 assert(result_ty.containerLayout(zcu) != .@"packed");
10843
10844 if (isByRef(result_ty, zcu)) {
10845 // TODO in debug builds init to undef so that the padding will be 0xaa
10846 // even if we fully populate the fields.
10847 const alignment = result_ty.abiAlignment(zcu).toLlvm();
10848 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10849
10850 for (elements, 0..) |elem, i| {
10851 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10852
10853 const llvm_elem = try self.resolveInst(elem);
10854 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10855 const field_ptr =
10856 try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
10857 const field_ptr_ty = try pt.ptrType(.{
10858 .child = self.typeOf(elem).toIntern(),
10859 .flags = .{
10860 .alignment = result_ty.fieldAlignment(i, zcu),
10861 },
10862 });
10863 try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
10864 }
10865
10866 return alloca_inst;
10867 } else {
10868 var result = try o.builder.poisonValue(llvm_result_ty);
10869 for (elements, 0..) |elem, i| {
10870 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10871
10872 const llvm_elem = try self.resolveInst(elem);
10873 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10874 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
10875 }
10876 return result;
10877 }
10878 },
10879 .array => {
10880 assert(isByRef(result_ty, zcu));
10881
10882 const llvm_usize = try o.lowerType(pt, Type.usize);
10883 const usize_zero = try o.builder.intValue(llvm_usize, 0);
10884 const alignment = result_ty.abiAlignment(zcu).toLlvm();
10885 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10886
10887 const array_info = result_ty.arrayInfo(zcu);
10888 const elem_ptr_ty = try pt.ptrType(.{
10889 .child = array_info.elem_type.toIntern(),
10890 });
10891
10892 for (elements, 0..) |elem, i| {
10893 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10894 usize_zero, try o.builder.intValue(llvm_usize, i),
10895 }, "");
10896 const llvm_elem = try self.resolveInst(elem);
10897 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
10898 }
10899 if (array_info.sentinel) |sent_val| {
10900 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10901 usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
10902 }, "");
10903 const llvm_elem = try self.resolveValue(sent_val);
10904 try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
10905 }
10906
10907 return alloca_inst;
10908 },
10909 else => unreachable,
10910 }
10911 }
10912
10913 fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10914 const o = self.ng.object;
10915 const pt = self.ng.pt;
10916 const zcu = pt.zcu;
10917 const ip = &zcu.intern_pool;
10918 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10919 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
10920 const union_ty = self.typeOfIndex(inst);
10921 const union_llvm_ty = try o.lowerType(pt, union_ty);
10922 const layout = union_ty.unionGetLayout(zcu);
10923 const union_obj = zcu.typeToUnion(union_ty).?;
10924
10925 if (union_obj.flagsUnordered(ip).layout == .@"packed") {
10926 const big_bits = union_ty.bitSize(zcu);
10927 const int_llvm_ty = try o.builder.intType(@intCast(big_bits));
10928 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
10929 const non_int_val = try self.resolveInst(extra.init);
10930 const small_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
10931 const small_int_val = if (field_ty.isPtrAtRuntime(zcu))
10932 try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
10933 else
10934 try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
10935 return self.wip.conv(.unsigned, small_int_val, int_llvm_ty, "");
10936 }
10937
10938 const tag_int_val = blk: {
10939 const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
10940 const union_field_name = union_obj.loadTagType(ip).names.get(ip)[extra.field_index];
10941 const enum_field_index = tag_ty.enumFieldIndex(union_field_name, zcu).?;
10942 const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);
10943 break :blk try tag_val.intFromEnum(tag_ty, pt);
10944 };
10945 if (layout.payload_size == 0) {
10946 if (layout.tag_size == 0) {
10947 return .none;
10948 }
10949 assert(!isByRef(union_ty, zcu));
10950 var big_int_space: Value.BigIntSpace = undefined;
10951 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
10952 return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
10953 }
10954 assert(isByRef(union_ty, zcu));
10955 // The llvm type of the alloca will be the named LLVM union type, and will not
10956 // necessarily match the format that we need, depending on which tag is active.
10957 // We must construct the correct unnamed struct type here, in order to then set
10958 // the fields appropriately.
10959 const alignment = layout.abi_align.toLlvm();
10960 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
10961 const llvm_payload = try self.resolveInst(extra.init);
10962 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
10963 const field_llvm_ty = try o.lowerType(pt, field_ty);
10964 const field_size = field_ty.abiSize(zcu);
10965 const field_align = union_ty.fieldAlignment(extra.field_index, zcu);
10966 const llvm_usize = try o.lowerType(pt, Type.usize);
10967 const usize_zero = try o.builder.intValue(llvm_usize, 0);
10968
10969 const llvm_union_ty = t: {
10970 const payload_ty = p: {
10971 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
10972 const padding_len = layout.payload_size;
10973 break :p try o.builder.arrayType(padding_len, .i8);
10974 }
10975 if (field_size == layout.payload_size) {
10976 break :p field_llvm_ty;
10977 }
10978 const padding_len = layout.payload_size - field_size;
10979 break :p try o.builder.structType(.@"packed", &.{
10980 field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
10981 });
10982 };
10983 if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
10984 const tag_ty = try o.lowerType(pt, Type.fromInterned(union_obj.enum_tag_ty));
10985 var fields: [3]Builder.Type = undefined;
10986 var fields_len: usize = 2;
10987 if (layout.tag_align.compare(.gte, layout.payload_align)) {
10988 fields = .{ tag_ty, payload_ty, undefined };
10989 } else {
10990 fields = .{ payload_ty, tag_ty, undefined };
10991 }
10992 if (layout.padding != 0) {
10993 fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
10994 fields_len += 1;
10995 }
10996 break :t try o.builder.structType(.normal, fields[0..fields_len]);
10997 };
10998
10999 // Now we follow the layout as expressed above with GEP instructions to set the
11000 // tag and the payload.
11001 const field_ptr_ty = try pt.ptrType(.{
11002 .child = field_ty.toIntern(),
11003 .flags = .{ .alignment = field_align },
11004 });
11005 if (layout.tag_size == 0) {
11006 const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
11007 const len: usize = if (field_size == layout.payload_size) 2 else 3;
11008 const field_ptr =
11009 try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
11010 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
11011 return result_ptr;
11012 }
11013
11014 {
11015 const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
11016 const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };
11017 const len: usize = if (field_size == layout.payload_size) 2 else 3;
11018 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
11019 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
11020 }
11021 {
11022 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
11023 const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };
11024 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");
11025 const tag_ty = try o.lowerType(pt, Type.fromInterned(union_obj.enum_tag_ty));
11026 var big_int_space: Value.BigIntSpace = undefined;
11027 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
11028 const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
11029 const tag_alignment = Type.fromInterned(union_obj.enum_tag_ty).abiAlignment(zcu).toLlvm();
11030 _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
11031 }
11032
11033 return result_ptr;
11034 }
11035
11036 fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
11037 const o = self.ng.object;
11038 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
11039
11040 comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0);
11041 comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1);
11042
11043 // TODO these two asserts should be able to be comptime because the type is a u2
11044 assert(prefetch.locality >= 0);
11045 assert(prefetch.locality <= 3);
11046
11047 comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0);
11048 comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1);
11049
11050 // LLVM fails during codegen of instruction cache prefetchs for these architectures.
11051 // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
11052 // by the target.
11053 // To work around this, don't emit llvm.prefetch in this case.
11054 // See https://bugs.llvm.org/show_bug.cgi?id=21037
11055 const zcu = self.ng.pt.zcu;
11056 const target = zcu.getTarget();
11057 switch (prefetch.cache) {
11058 .instruction => switch (target.cpu.arch) {
11059 .x86_64,
11060 .x86,
11061 .powerpc,
11062 .powerpcle,
11063 .powerpc64,
11064 .powerpc64le,
11065 => return .none,
11066 .arm, .armeb, .thumb, .thumbeb => {
11067 switch (prefetch.rw) {
11068 .write => return .none,
11069 else => {},
11070 }
11071 },
11072 else => {},
11073 },
11074 .data => {},
11075 }
11076
11077 _ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
11078 try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
11079 try o.builder.intValue(.i32, prefetch.rw),
11080 try o.builder.intValue(.i32, prefetch.locality),
11081 try o.builder.intValue(.i32, prefetch.cache),
11082 }, "");
11083 return .none;
11084 }
11085
11086 fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
11087 const o = self.ng.object;
11088 const pt = self.ng.pt;
11089 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
11090 const inst_ty = self.typeOfIndex(inst);
11091 const operand = try self.resolveInst(ty_op.operand);
11092
11093 return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), "");
11094 }
11095
11096 fn workIntrinsic(
11097 self: *FuncGen,
11098 dimension: u32,
11099 default: u32,
11100 comptime basename: []const u8,
11101 ) !Builder.Value {
11102 return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
11103 0 => @field(Builder.Intrinsic, basename ++ ".x"),
11104 1 => @field(Builder.Intrinsic, basename ++ ".y"),
11105 2 => @field(Builder.Intrinsic, basename ++ ".z"),
11106 else => return self.ng.object.builder.intValue(.i32, default),
11107 }, &.{}, &.{}, "");
11108 }
11109
11110 fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
11111 const target = self.ng.pt.zcu.getTarget();
11112
11113 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
11114 const dimension = pl_op.payload;
11115
11116 return switch (target.cpu.arch) {
11117 .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
11118 .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
11119 else => unreachable,
11120 };
11121 }
11122
11123 fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
11124 const o = self.ng.object;
11125 const pt = self.ng.pt;
11126 const target = pt.zcu.getTarget();
11127
11128 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
11129 const dimension = pl_op.payload;
11130
11131 switch (target.cpu.arch) {
11132 .amdgcn => {
11133 if (dimension >= 3) return .@"1";
11134
11135 // Fetch the dispatch pointer, which points to this structure:
11136 // https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
11137 const dispatch_ptr =
11138 try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
11139
11140 // Load the work_group_* member from the struct as u16.
11141 // Just treat the dispatch pointer as an array of u16 to keep things simple.
11142 const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{
11143 try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),
11144 }, "");
11145 const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
11146 return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
11147 },
11148 .nvptx, .nvptx64 => {
11149 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
11150 },
11151 else => unreachable,
11152 }
11153 }
11154
11155 fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
11156 const target = self.ng.pt.zcu.getTarget();
11157
11158 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
11159 const dimension = pl_op.payload;
11160
11161 return switch (target.cpu.arch) {
11162 .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
11163 .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
11164 else => unreachable,
11165 };
11166 }
11167
11168 fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
11169 const o = self.ng.object;
11170 const pt = self.ng.pt;
11171
11172 const table = o.error_name_table;
11173 if (table != .none) return table;
11174
11175 // TODO: Address space
11176 const variable_index =
11177 try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default);
11178 variable_index.setLinkage(.private, &o.builder);
11179 variable_index.setMutability(.constant, &o.builder);
11180 variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
11181 variable_index.setAlignment(
11182 Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(),
11183 &o.builder,
11184 );
11185
11186 o.error_name_table = variable_index;
11187 return variable_index;
11188 }
11189
11190 /// Assumes the optional is not pointer-like and payload has bits.
11191 fn optCmpNull(
11192 self: *FuncGen,
11193 cond: Builder.IntegerCondition,
11194 opt_llvm_ty: Builder.Type,
11195 opt_handle: Builder.Value,
11196 is_by_ref: bool,
11197 access_kind: Builder.MemoryAccessKind,
11198 ) Allocator.Error!Builder.Value {
11199 const o = self.ng.object;
11200 const field = b: {
11201 if (is_by_ref) {
11202 const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
11203 break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
11204 }
11205 break :b try self.wip.extractValue(opt_handle, &.{1}, "");
11206 };
11207 comptime assert(optional_layout_version == 3);
11208
11209 return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");
11210 }
11211
11212 /// Assumes the optional is not pointer-like and payload has bits.
11213 fn optPayloadHandle(
11214 fg: *FuncGen,
11215 opt_llvm_ty: Builder.Type,
11216 opt_handle: Builder.Value,
11217 opt_ty: Type,
11218 can_elide_load: bool,
11219 ) !Builder.Value {
11220 const pt = fg.ng.pt;
11221 const zcu = pt.zcu;
11222 const payload_ty = opt_ty.optionalChild(zcu);
11223
11224 if (isByRef(opt_ty, zcu)) {
11225 // We have a pointer and we need to return a pointer to the first field.
11226 const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
11227
11228 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
11229 if (isByRef(payload_ty, zcu)) {
11230 if (can_elide_load)
11231 return payload_ptr;
11232
11233 return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
11234 }
11235 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);
11236 }
11237
11238 assert(!isByRef(payload_ty, zcu));
11239 return fg.wip.extractValue(opt_handle, &.{0}, "");
11240 }
11241
11242 fn buildOptional(
11243 self: *FuncGen,
11244 optional_ty: Type,
11245 payload: Builder.Value,
11246 non_null_bit: Builder.Value,
11247 ) !Builder.Value {
11248 const o = self.ng.object;
11249 const pt = self.ng.pt;
11250 const zcu = pt.zcu;
11251 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
11252 const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
11253
11254 if (isByRef(optional_ty, zcu)) {
11255 const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
11256 const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
11257
11258 {
11259 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
11260 _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
11261 }
11262 {
11263 const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
11264 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
11265 _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
11266 }
11267
11268 return alloca_inst;
11269 }
11270
11271 return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
11272 }
11273
11274 fn fieldPtr(
11275 self: *FuncGen,
11276 inst: Air.Inst.Index,
11277 struct_ptr: Builder.Value,
11278 struct_ptr_ty: Type,
11279 field_index: u32,
11280 ) !Builder.Value {
11281 const o = self.ng.object;
11282 const pt = self.ng.pt;
11283 const zcu = pt.zcu;
11284 const struct_ty = struct_ptr_ty.childType(zcu);
11285 switch (struct_ty.zigTypeTag(zcu)) {
11286 .@"struct" => switch (struct_ty.containerLayout(zcu)) {
11287 .@"packed" => {
11288 const result_ty = self.typeOfIndex(inst);
11289 const result_ty_info = result_ty.ptrInfo(zcu);
11290 const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);
11291 const struct_type = zcu.typeToStruct(struct_ty).?;
11292
11293 if (result_ty_info.packed_offset.host_size != 0) {
11294 // From LLVM's perspective, a pointer to a packed struct and a pointer
11295 // to a field of a packed struct are the same. The difference is in the
11296 // Zig pointer type which provides information for how to mask and shift
11297 // out the relevant bits when accessing the pointee.
11298 return struct_ptr;
11299 }
11300
11301 // We have a pointer to a packed struct field that happens to be byte-aligned.
11302 // Offset our operand pointer by the correct number of bytes.
11303 const byte_offset = @divExact(zcu.structPackedFieldBitOffset(struct_type, field_index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);
11304 if (byte_offset == 0) return struct_ptr;
11305 const usize_ty = try o.lowerType(pt, Type.usize);
11306 const llvm_index = try o.builder.intValue(usize_ty, byte_offset);
11307 return self.wip.gep(.inbounds, .i8, struct_ptr, &.{llvm_index}, "");
11308 },
11309 else => {
11310 const struct_llvm_ty = try o.lowerPtrElemTy(pt, struct_ty);
11311
11312 if (o.llvmFieldIndex(struct_ty, field_index)) |llvm_field_index| {
11313 return self.wip.gepStruct(struct_llvm_ty, struct_ptr, llvm_field_index, "");
11314 } else {
11315 // If we found no index then this means this is a zero sized field at the
11316 // end of the struct. Treat our struct pointer as an array of two and get
11317 // the index to the element at index `1` to get a pointer to the end of
11318 // the struct.
11319 const llvm_index = try o.builder.intValue(
11320 try o.lowerType(pt, Type.usize),
11321 @intFromBool(struct_ty.hasRuntimeBitsIgnoreComptime(zcu)),
11322 );
11323 return self.wip.gep(.inbounds, struct_llvm_ty, struct_ptr, &.{llvm_index}, "");
11324 }
11325 },
11326 },
11327 .@"union" => {
11328 const layout = struct_ty.unionGetLayout(zcu);
11329 if (layout.payload_size == 0 or struct_ty.containerLayout(zcu) == .@"packed") return struct_ptr;
11330 const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
11331 const union_llvm_ty = try o.lowerType(pt, struct_ty);
11332 return self.wip.gepStruct(union_llvm_ty, struct_ptr, payload_index, "");
11333 },
11334 else => unreachable,
11335 }
11336 }
11337
11338 /// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
11339 fn loadTruncate(
11340 fg: *FuncGen,
11341 access_kind: Builder.MemoryAccessKind,
11342 payload_ty: Type,
11343 payload_ptr: Builder.Value,
11344 payload_alignment: Builder.Alignment,
11345 ) !Builder.Value {
11346 // from https://llvm.org/docs/LangRef.html#load-instruction :
11347 // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
11348 // => so load the byte aligned value and trunc the unwanted bits.
11349
11350 const o = fg.ng.object;
11351 const pt = fg.ng.pt;
11352 const zcu = pt.zcu;
11353 const payload_llvm_ty = try o.lowerType(pt, payload_ty);
11354 const abi_size = payload_ty.abiSize(zcu);
11355
11356 const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
11357 try o.builder.intType(@intCast(abi_size * 8))
11358 else
11359 payload_llvm_ty;
11360 const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
11361 const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big)
11362 try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
11363 load_llvm_ty,
11364 (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
11365 ), "")
11366 else
11367 loaded;
11368
11369 return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
11370 }
11371
11372 /// Load a by-ref type by constructing a new alloca and performing a memcpy.
11373 fn loadByRef(
11374 fg: *FuncGen,
11375 ptr: Builder.Value,
11376 pointee_type: Type,
11377 ptr_alignment: Builder.Alignment,
11378 access_kind: Builder.MemoryAccessKind,
11379 ) !Builder.Value {
11380 const o = fg.ng.object;
11381 const pt = fg.ng.pt;
11382 const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
11383 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
11384 .max(pointee_type.abiAlignment(pt.zcu)).toLlvm();
11385 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
11386 const size_bytes = pointee_type.abiSize(pt.zcu);
11387 _ = try fg.wip.callMemCpy(
11388 result_ptr,
11389 result_align,
11390 ptr,
11391 ptr_alignment,
11392 try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes),
11393 access_kind,
11394 fg.disable_intrinsics,
11395 );
11396 return result_ptr;
11397 }
11398
11399 /// This function always performs a copy. For isByRef=true types, it creates a new
11400 /// alloca and copies the value into it, then returns the alloca instruction.
11401 /// For isByRef=false types, it creates a load instruction and returns it.
11402 fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value {
11403 const o = self.ng.object;
11404 const pt = self.ng.pt;
11405 const zcu = pt.zcu;
11406 const info = ptr_ty.ptrInfo(zcu);
11407 const elem_ty = Type.fromInterned(info.child);
11408 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) return .none;
11409
11410 const ptr_alignment = (if (info.flags.alignment != .none)
11411 @as(InternPool.Alignment, info.flags.alignment)
11412 else
11413 elem_ty.abiAlignment(zcu)).toLlvm();
11414
11415 const access_kind: Builder.MemoryAccessKind =
11416 if (info.flags.is_volatile) .@"volatile" else .normal;
11417
11418 if (info.flags.vector_index != .none) {
11419 const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11420 const vec_elem_ty = try o.lowerType(pt, elem_ty);
11421 const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11422
11423 const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, "");
11424 return self.wip.extractElement(loaded_vector, index_u32, "");
11425 }
11426
11427 if (info.packed_offset.host_size == 0) {
11428 if (isByRef(elem_ty, zcu)) {
11429 return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
11430 }
11431 return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
11432 }
11433
11434 const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11435 const containing_int =
11436 try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, "");
11437
11438 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11439 const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset);
11440 const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
11441 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
11442
11443 if (isByRef(elem_ty, zcu)) {
11444 const result_align = elem_ty.abiAlignment(zcu).toLlvm();
11445 const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);
11446
11447 const same_size_int = try o.builder.intType(@intCast(elem_bits));
11448 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11449 _ = try self.wip.store(.normal, truncated_int, result_ptr, result_align);
11450 return result_ptr;
11451 }
11452
11453 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
11454 const same_size_int = try o.builder.intType(@intCast(elem_bits));
11455 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11456 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
11457 }
11458
11459 if (elem_ty.isPtrAtRuntime(zcu)) {
11460 const same_size_int = try o.builder.intType(@intCast(elem_bits));
11461 const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11462 return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
11463 }
11464
11465 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
11466 }
11467
11468 fn store(
11469 self: *FuncGen,
11470 ptr: Builder.Value,
11471 ptr_ty: Type,
11472 elem: Builder.Value,
11473 ordering: Builder.AtomicOrdering,
11474 ) !void {
11475 const o = self.ng.object;
11476 const pt = self.ng.pt;
11477 const zcu = pt.zcu;
11478 const info = ptr_ty.ptrInfo(zcu);
11479 const elem_ty = Type.fromInterned(info.child);
11480 if (!elem_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) {
11481 return;
11482 }
11483 const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
11484 const access_kind: Builder.MemoryAccessKind =
11485 if (info.flags.is_volatile) .@"volatile" else .normal;
11486
11487 if (info.flags.vector_index != .none) {
11488 const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11489 const vec_elem_ty = try o.lowerType(pt, elem_ty);
11490 const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11491
11492 const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
11493
11494 const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
11495
11496 assert(ordering == .none);
11497 _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
11498 return;
11499 }
11500
11501 if (info.packed_offset.host_size != 0) {
11502 const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11503 assert(ordering == .none);
11504 const containing_int =
11505 try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
11506 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11507 const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
11508 // Convert to equally-sized integer type in order to perform the bit
11509 // operations on the value to store
11510 const value_bits_type = try o.builder.intType(@intCast(elem_bits));
11511 const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
11512 try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
11513 else
11514 try self.wip.cast(.bitcast, elem, value_bits_type, "");
11515
11516 const mask_val = blk: {
11517 const zext = try self.wip.cast(
11518 .zext,
11519 try o.builder.intValue(value_bits_type, -1),
11520 containing_int_ty,
11521 "",
11522 );
11523 const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
11524 break :blk try self.wip.bin(
11525 .xor,
11526 shl,
11527 try o.builder.intValue(containing_int_ty, -1),
11528 "",
11529 );
11530 };
11531
11532 const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
11533 const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
11534 const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
11535 const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
11536
11537 assert(ordering == .none);
11538 _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
11539 return;
11540 }
11541 if (!isByRef(elem_ty, zcu)) {
11542 _ = try self.wip.storeAtomic(
11543 access_kind,
11544 elem,
11545 ptr,
11546 self.sync_scope,
11547 ordering,
11548 ptr_alignment,
11549 );
11550 return;
11551 }
11552 assert(ordering == .none);
11553 _ = try self.wip.callMemCpy(
11554 ptr,
11555 ptr_alignment,
11556 elem,
11557 elem_ty.abiAlignment(zcu).toLlvm(),
11558 try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)),
11559 access_kind,
11560 self.disable_intrinsics,
11561 );
11562 }
11563
11564 fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
11565 const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
11566 const o = fg.ng.object;
11567 const pt = fg.ng.pt;
11568 const usize_ty = try o.lowerType(pt, Type.usize);
11569 const zero = try o.builder.intValue(usize_ty, 0);
11570 const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
11571 const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
11572 _ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
11573 }
11574
11575 fn valgrindClientRequest(
11576 fg: *FuncGen,
11577 default_value: Builder.Value,
11578 request: Builder.Value,
11579 a1: Builder.Value,
11580 a2: Builder.Value,
11581 a3: Builder.Value,
11582 a4: Builder.Value,
11583 a5: Builder.Value,
11584 ) Allocator.Error!Builder.Value {
11585 const o = fg.ng.object;
11586 const pt = fg.ng.pt;
11587 const zcu = pt.zcu;
11588 const target = zcu.getTarget();
11589 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
11590
11591 const llvm_usize = try o.lowerType(pt, Type.usize);
11592 const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
11593
11594 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
11595 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
11596 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
11597 fg.valgrind_client_request_array = array_ptr;
11598 break :a array_ptr;
11599 } else fg.valgrind_client_request_array;
11600 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
11601 const zero = try o.builder.intValue(llvm_usize, 0);
11602 for (array_elements, 0..) |elem, i| {
11603 const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{
11604 zero, try o.builder.intValue(llvm_usize, i),
11605 }, "");
11606 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
11607 }
11608
11609 const arch_specific: struct {
11610 template: [:0]const u8,
11611 constraints: [:0]const u8,
11612 } = switch (target.cpu.arch) {
11613 .arm, .armeb, .thumb, .thumbeb => .{
11614 .template =
11615 \\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
11616 \\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
11617 \\ orr r10, r10, r10
11618 ,
11619 .constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11620 },
11621 .aarch64, .aarch64_be => .{
11622 .template =
11623 \\ ror x12, x12, #3 ; ror x12, x12, #13
11624 \\ ror x12, x12, #51 ; ror x12, x12, #61
11625 \\ orr x10, x10, x10
11626 ,
11627 .constraints = "={x3},{x4},{x3},~{cc},~{memory}",
11628 },
11629 .mips, .mipsel => .{
11630 .template =
11631 \\ srl $$0, $$0, 13
11632 \\ srl $$0, $$0, 29
11633 \\ srl $$0, $$0, 3
11634 \\ srl $$0, $$0, 19
11635 \\ or $$13, $$13, $$13
11636 ,
11637 .constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11638 },
11639 .mips64, .mips64el => .{
11640 .template =
11641 \\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
11642 \\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
11643 \\ or $$13, $$13, $$13
11644 ,
11645 .constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11646 },
11647 .powerpc, .powerpcle => .{
11648 .template =
11649 \\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
11650 \\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
11651 \\ or 1, 1, 1
11652 ,
11653 .constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11654 },
11655 .powerpc64, .powerpc64le => .{
11656 .template =
11657 \\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
11658 \\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
11659 \\ or 1, 1, 1
11660 ,
11661 .constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11662 },
11663 .riscv64 => .{
11664 .template =
11665 \\ .option push
11666 \\ .option norvc
11667 \\ srli zero, zero, 3
11668 \\ srli zero, zero, 13
11669 \\ srli zero, zero, 51
11670 \\ srli zero, zero, 61
11671 \\ or a0, a0, a0
11672 \\ .option pop
11673 ,
11674 .constraints = "={a3},{a4},{a3},~{cc},~{memory}",
11675 },
11676 .s390x => .{
11677 .template =
11678 \\ lr %r15, %r15
11679 \\ lr %r1, %r1
11680 \\ lr %r2, %r2
11681 \\ lr %r3, %r3
11682 \\ lr %r2, %r2
11683 ,
11684 .constraints = "={r3},{r2},{r3},~{cc},~{memory}",
11685 },
11686 .x86 => .{
11687 .template =
11688 \\ roll $$3, %edi ; roll $$13, %edi
11689 \\ roll $$61, %edi ; roll $$51, %edi
11690 \\ xchgl %ebx, %ebx
11691 ,
11692 .constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11693 },
11694 .x86_64 => .{
11695 .template =
11696 \\ rolq $$3, %rdi ; rolq $$13, %rdi
11697 \\ rolq $$61, %rdi ; rolq $$51, %rdi
11698 \\ xchgq %rbx, %rbx
11699 ,
11700 .constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11701 },
11702 else => unreachable,
11703 };
11704
11705 return fg.wip.callAsm(
11706 .none,
11707 try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
11708 .{ .sideeffect = true },
11709 try o.builder.string(arch_specific.template),
11710 try o.builder.string(arch_specific.constraints),
11711 &.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
11712 "",
11713 );
11714 }
11715
11716 fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
11717 const zcu = fg.ng.pt.zcu;
11718 return fg.air.typeOf(inst, &zcu.intern_pool);
11719 }
11720
11721 fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
11722 const zcu = fg.ng.pt.zcu;
11723 return fg.air.typeOfIndex(inst, &zcu.intern_pool);
11724 }
11725};
11726
11727fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering {
11728 return switch (atomic_order) {
11729 .unordered => .unordered,
11730 .monotonic => .monotonic,
11731 .acquire => .acquire,
11732 .release => .release,
11733 .acq_rel => .acq_rel,
11734 .seq_cst => .seq_cst,
11735 };
11736}
11737
11738fn toLlvmAtomicRmwBinOp(
11739 op: std.builtin.AtomicRmwOp,
11740 is_signed: bool,
11741 is_float: bool,
11742) Builder.Function.Instruction.AtomicRmw.Operation {
11743 return switch (op) {
11744 .Xchg => .xchg,
11745 .Add => if (is_float) .fadd else return .add,
11746 .Sub => if (is_float) .fsub else return .sub,
11747 .And => .@"and",
11748 .Nand => .nand,
11749 .Or => .@"or",
11750 .Xor => .xor,
11751 .Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
11752 .Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
11753 };
11754}
11755
11756const CallingConventionInfo = struct {
11757 /// The LLVM calling convention to use.
11758 llvm_cc: Builder.CallConv,
11759 /// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
11760 align_stack: bool,
11761 /// Whether the function needs a `naked` attribute.
11762 naked: bool,
11763 /// How many leading parameters to apply the `inreg` attribute to.
11764 inreg_param_count: u2 = 0,
11765};
11766
11767pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
11768 const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
11769 const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
11770 inline else => |pl| switch (@TypeOf(pl)) {
11771 void => .{ null, 0 },
11772 std.builtin.CallingConvention.ArcInterruptOptions,
11773 std.builtin.CallingConvention.ArmInterruptOptions,
11774 std.builtin.CallingConvention.RiscvInterruptOptions,
11775 std.builtin.CallingConvention.ShInterruptOptions,
11776 std.builtin.CallingConvention.MicroblazeInterruptOptions,
11777 std.builtin.CallingConvention.MipsInterruptOptions,
11778 std.builtin.CallingConvention.CommonOptions,
11779 => .{ pl.incoming_stack_alignment, 0 },
11780 std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
11781 else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
11782 },
11783 };
11784 return .{
11785 .llvm_cc = llvm_cc,
11786 .align_stack = if (incoming_stack_alignment) |a| need_align: {
11787 const normal_stack_align = target.stackAlignment();
11788 break :need_align a < normal_stack_align;
11789 } else false,
11790 .naked = cc == .naked,
11791 .inreg_param_count = register_params,
11792 };
11793}
11794fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
11795 if (target.cCallingConvention()) |default_c| {
11796 if (cc_tag == default_c) {
11797 return .ccc;
11798 }
11799 }
11800 return switch (cc_tag) {
11801 .@"inline" => unreachable,
11802 .auto, .async => .fastcc,
11803 .naked => .ccc,
11804 .x86_64_sysv => .x86_64_sysvcc,
11805 .x86_64_win => .win64cc,
11806 .x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
11807 .x86_regcallcc
11808 else
11809 null,
11810 .x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
11811 .x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11812 else
11813 null,
11814 .x86_64_vectorcall => .x86_vectorcallcc,
11815 .x86_64_interrupt => .x86_intrcc,
11816 .x86_stdcall => .x86_stdcallcc,
11817 .x86_fastcall => .x86_fastcallcc,
11818 .x86_thiscall => .x86_thiscallcc,
11819 .x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
11820 .x86_regcallcc
11821 else
11822 null,
11823 .x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
11824 .x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11825 else
11826 null,
11827 .x86_vectorcall => .x86_vectorcallcc,
11828 .x86_interrupt => .x86_intrcc,
11829 .aarch64_vfabi => .aarch64_vector_pcs,
11830 .aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
11831 .arm_aapcs => .arm_aapcscc,
11832 .arm_aapcs_vfp => .arm_aapcs_vfpcc,
11833 .riscv64_lp64_v => .riscv_vectorcallcc,
11834 .riscv32_ilp32_v => .riscv_vectorcallcc,
11835 .avr_builtin => .avr_builtincc,
11836 .avr_signal => .avr_signalcc,
11837 .avr_interrupt => .avr_intrcc,
11838 .m68k_rtd => .m68k_rtdcc,
11839 .m68k_interrupt => .m68k_intrcc,
11840 .msp430_interrupt => .msp430_intrcc,
11841 .amdgcn_kernel => .amdgpu_kernel,
11842 .amdgcn_cs => .amdgpu_cs,
11843 .nvptx_device => .ptx_device,
11844 .nvptx_kernel => .ptx_kernel,
11845
11846 // Calling conventions which LLVM uses function attributes for.
11847 .riscv64_interrupt,
11848 .riscv32_interrupt,
11849 .arm_interrupt,
11850 .mips64_interrupt,
11851 .mips_interrupt,
11852 .csky_interrupt,
11853 => .ccc,
11854
11855 // All the calling conventions which LLVM does not have a general representation for.
11856 // Note that these are often still supported through the `cCallingConvention` path above via `ccc`.
11857 .x86_16_cdecl,
11858 .x86_16_stdcall,
11859 .x86_16_regparmcall,
11860 .x86_16_interrupt,
11861 .x86_sysv,
11862 .x86_win,
11863 .x86_thiscall_mingw,
11864 .x86_64_x32,
11865 .aarch64_aapcs,
11866 .aarch64_aapcs_darwin,
11867 .aarch64_aapcs_win,
11868 .alpha_osf,
11869 .microblaze_std,
11870 .microblaze_interrupt,
11871 .mips64_n64,
11872 .mips64_n32,
11873 .mips_o32,
11874 .riscv64_lp64,
11875 .riscv32_ilp32,
11876 .sparc64_sysv,
11877 .sparc_sysv,
11878 .powerpc64_elf,
11879 .powerpc64_elf_altivec,
11880 .powerpc64_elf_v2,
11881 .powerpc_sysv,
11882 .powerpc_sysv_altivec,
11883 .powerpc_aix,
11884 .powerpc_aix_altivec,
11885 .wasm_mvp,
11886 .arc_sysv,
11887 .arc_interrupt,
11888 .avr_gnu,
11889 .bpf_std,
11890 .csky_sysv,
11891 .hexagon_sysv,
11892 .hexagon_sysv_hvx,
11893 .hppa_elf,
11894 .hppa64_elf,
11895 .kvx_lp64,
11896 .kvx_ilp32,
11897 .lanai_sysv,
11898 .loongarch64_lp64,
11899 .loongarch32_ilp32,
11900 .m68k_sysv,
11901 .m68k_gnu,
11902 .msp430_eabi,
11903 .or1k_sysv,
11904 .propeller_sysv,
11905 .s390x_sysv,
11906 .s390x_sysv_vx,
11907 .sh_gnu,
11908 .sh_renesas,
11909 .sh_interrupt,
11910 .ve_sysv,
11911 .xcore_xs1,
11912 .xcore_xs2,
11913 .xtensa_call0,
11914 .xtensa_windowed,
11915 .amdgcn_device,
11916 .spirv_device,
11917 .spirv_kernel,
11918 .spirv_fragment,
11919 .spirv_vertex,
11920 => null,
11921 };
11922}
11923
11924/// Convert a zig-address space to an llvm address space.
11925fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
11926 for (llvmAddrSpaceInfo(target)) |info| if (info.zig == address_space) return info.llvm;
11927 unreachable;
11928}
11929
11930const AddrSpaceInfo = struct {
11931 zig: ?std.builtin.AddressSpace,
11932 llvm: Builder.AddrSpace,
11933 non_integral: bool = false,
11934 size: ?u16 = null,
11935 abi: ?u16 = null,
11936 pref: ?u16 = null,
11937 idx: ?u16 = null,
11938 force_in_data_layout: bool = false,
11939};
11940fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {
11941 return switch (target.cpu.arch) {
11942 .x86, .x86_64 => &.{
11943 .{ .zig = .generic, .llvm = .default },
11944 .{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs },
11945 .{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs },
11946 .{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss },
11947 .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
11948 .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
11949 .{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
11950 },
11951 .nvptx, .nvptx64 => &.{
11952 .{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic },
11953 .{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global },
11954 .{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant },
11955 .{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param },
11956 .{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared },
11957 .{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local },
11958 },
11959 .amdgcn => &.{
11960 .{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true },
11961 .{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true },
11962 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 },
11963 .{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 },
11964 .{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true },
11965 .{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 },
11966 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 },
11967 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 },
11968 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 },
11969 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 },
11970 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 },
11971 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 },
11972 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 },
11973 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 },
11974 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 },
11975 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 },
11976 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 },
11977 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 },
11978 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 },
11979 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 },
11980 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 },
11981 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 },
11982 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 },
11983 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 },
11984 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 },
11985 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 },
11986 .{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register },
11987 },
11988 .avr => &.{
11989 .{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 },
11990 .{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 },
11991 .{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 },
11992 .{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 },
11993 .{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 },
11994 .{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 },
11995 .{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 },
11996 },
11997 .wasm32, .wasm64 => &.{
11998 .{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true },
11999 .{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true },
12000 .{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
12001 .{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
12002 },
12003 .m68k => &.{
12004 .{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 },
12005 },
12006 else => &.{
12007 .{ .zig = .generic, .llvm = .default },
12008 },
12009 };
12010}
12011
12012/// On some targets, local values that are in the generic address space must be generated into a
12013/// different address, space and then cast back to the generic address space.
12014/// For example, on GPUs local variable declarations must be generated into the local address space.
12015/// This function returns the address space local values should be generated into.
12016fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
12017 return switch (target.cpu.arch) {
12018 // On amdgcn, locals should be generated into the private address space.
12019 // To make Zig not impossible to use, these are then converted to addresses in the
12020 // generic address space and treates as regular pointers. This is the way that HIP also does it.
12021 .amdgcn => Builder.AddrSpace.amdgpu.private,
12022 else => .default,
12023 };
12024}
12025
12026/// On some targets, global values that are in the generic address space must be generated into a
12027/// different address space, and then cast back to the generic address space.
12028fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {
12029 return switch (target.cpu.arch) {
12030 // On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
12031 // them.
12032 .amdgcn => Builder.AddrSpace.amdgpu.global,
12033 else => .default,
12034 };
12035}
12036
12037/// Return the actual address space that a value should be stored in if its a global address space.
12038/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
12039fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
12040 return switch (wanted_address_space) {
12041 .generic => llvmDefaultGlobalAddressSpace(target),
12042 else => |as| toLlvmAddressSpace(as, target),
12043 };
12044}
12045
12046fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool {
12047 if (isByRef(ty, zcu)) {
12048 return true;
12049 } else if (target.cpu.arch.isX86() and
12050 !target.cpu.has(.x86, .evex512) and
12051 ty.totalVectorBits(zcu) >= 512)
12052 {
12053 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
12054 // "512-bit vector arguments require 'evex512' for AVX512"
12055 return true;
12056 } else {
12057 return false;
12058 }
12059}
12060
12061fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool {
12062 const return_type = Type.fromInterned(fn_info.return_type);
12063 if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) return false;
12064
12065 return switch (fn_info.cc) {
12066 .auto => returnTypeByRef(zcu, target, return_type),
12067 .x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
12068 .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
12069 .x86_sysv, .x86_win => isByRef(return_type, zcu),
12070 .x86_stdcall => !isScalar(zcu, return_type),
12071 .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
12072 .aarch64_aapcs,
12073 .aarch64_aapcs_darwin,
12074 .aarch64_aapcs_win,
12075 => aarch64_c_abi.classifyType(return_type, zcu) == .memory,
12076 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
12077 .memory, .i64_array => true,
12078 .i32_array => |size| size != 1,
12079 .byval => false,
12080 },
12081 .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
12082 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
12083 .memory, .i32_array => true,
12084 .byval => false,
12085 },
12086 else => false, // TODO: investigate other targets/callconvs
12087 };
12088}
12089
12090fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool {
12091 const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
12092 if (class[0] == .memory) return true;
12093 if (class[0] == .x87 and class[2] != .none) return true;
12094 return false;
12095}
12096
12097/// In order to support the C calling convention, some return types need to be lowered
12098/// completely differently in the function prototype to honor the C ABI, and then
12099/// be effectively bitcasted to the actual return type.
12100fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
12101 const zcu = pt.zcu;
12102 const return_type = Type.fromInterned(fn_info.return_type);
12103 if (!return_type.hasRuntimeBitsIgnoreComptime(zcu)) {
12104 // If the return type is an error set or an error union, then we make this
12105 // anyerror return type instead, so that it can be coerced into a function
12106 // pointer type which has anyerror as the return type.
12107 return if (return_type.isError(zcu)) try o.errorIntType(pt) else .void;
12108 }
12109 const target = zcu.getTarget();
12110 switch (fn_info.cc) {
12111 .@"inline" => unreachable,
12112 .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type),
12113
12114 .x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info),
12115 .x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info),
12116 .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void,
12117 .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type),
12118 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
12119 .memory => return .void,
12120 .float_array => return o.lowerType(pt, return_type),
12121 .byval => return o.lowerType(pt, return_type),
12122 .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
12123 .double_integer => return o.builder.arrayType(2, .i64),
12124 },
12125 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
12126 .memory, .i64_array => return .void,
12127 .i32_array => |len| return if (len == 1) .i32 else .void,
12128 .byval => return o.lowerType(pt, return_type),
12129 },
12130 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
12131 .memory, .i32_array => return .void,
12132 .byval => return o.lowerType(pt, return_type),
12133 },
12134 .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
12135 .memory => return .void,
12136 .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
12137 .double_integer => {
12138 const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
12139 .riscv64, .riscv64be => .i64,
12140 .riscv32, .riscv32be => .i32,
12141 else => unreachable,
12142 };
12143 return o.builder.structType(.normal, &.{ integer, integer });
12144 },
12145 .byval => return o.lowerType(pt, return_type),
12146 .fields => {
12147 var types_len: usize = 0;
12148 var types: [8]Builder.Type = undefined;
12149 for (0..return_type.structFieldCount(zcu)) |field_index| {
12150 const field_ty = return_type.fieldType(field_index, zcu);
12151 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
12152 types[types_len] = try o.lowerType(pt, field_ty);
12153 types_len += 1;
12154 }
12155 return o.builder.structType(.normal, types[0..types_len]);
12156 },
12157 },
12158 .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
12159 .direct => |scalar_ty| return o.lowerType(pt, scalar_ty),
12160 .indirect => return .void,
12161 },
12162 // TODO investigate other callconvs
12163 else => return o.lowerType(pt, return_type),
12164 }
12165}
12166
12167fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
12168 const zcu = pt.zcu;
12169 const return_type = Type.fromInterned(fn_info.return_type);
12170 switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
12171 .integer => {
12172 if (isScalar(zcu, return_type)) {
12173 return o.lowerType(pt, return_type);
12174 } else {
12175 return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
12176 }
12177 },
12178 .win_i128 => return o.builder.vectorType(.normal, 2, .i64),
12179 .memory => return .void,
12180 .sse => return o.lowerType(pt, return_type),
12181 else => unreachable,
12182 }
12183}
12184
12185fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
12186 const zcu = pt.zcu;
12187 const ip = &zcu.intern_pool;
12188 const return_type = Type.fromInterned(fn_info.return_type);
12189 if (isScalar(zcu, return_type)) {
12190 return o.lowerType(pt, return_type);
12191 }
12192 const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
12193 var types_index: u32 = 0;
12194 var types_buffer: [8]Builder.Type = undefined;
12195 for (classes) |class| {
12196 switch (class) {
12197 .integer => {
12198 types_buffer[types_index] = .i64;
12199 types_index += 1;
12200 },
12201 .sse => {
12202 types_buffer[types_index] = .double;
12203 types_index += 1;
12204 },
12205 .sseup => {
12206 if (types_buffer[types_index - 1] == .double) {
12207 types_buffer[types_index - 1] = .fp128;
12208 } else {
12209 types_buffer[types_index] = .double;
12210 types_index += 1;
12211 }
12212 },
12213 .float => {
12214 types_buffer[types_index] = .float;
12215 types_index += 1;
12216 },
12217 .float_combine => {
12218 types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
12219 types_index += 1;
12220 },
12221 .x87 => {
12222 if (types_index != 0 or classes[2] != .none) return .void;
12223 types_buffer[types_index] = .x86_fp80;
12224 types_index += 1;
12225 },
12226 .x87up => continue,
12227 .none => break,
12228 .memory, .integer_per_element => return .void,
12229 .win_i128 => unreachable, // windows only
12230 }
12231 }
12232 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12233 if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12234 assert(first_non_integer orelse classes.len == types_index);
12235 switch (ip.indexToKey(return_type.toIntern())) {
12236 .struct_type => {
12237 const struct_type = ip.loadStructType(return_type.toIntern());
12238 assert(struct_type.haveLayout(ip));
12239 const size: u64 = struct_type.sizeUnordered(ip);
12240 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12241 if (size % 8 > 0) {
12242 types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
12243 }
12244 },
12245 else => {},
12246 }
12247 if (types_index == 1) return types_buffer[0];
12248 }
12249 return o.builder.structType(.normal, types_buffer[0..types_index]);
12250}
12251
12252const ParamTypeIterator = struct {
12253 object: *Object,
12254 pt: Zcu.PerThread,
12255 fn_info: InternPool.Key.FuncType,
12256 zig_index: u32,
12257 llvm_index: u32,
12258 types_len: u32,
12259 types_buffer: [8]Builder.Type,
12260 byval_attr: bool,
12261
12262 const Lowering = union(enum) {
12263 no_bits,
12264 byval,
12265 byref,
12266 byref_mut,
12267 abi_sized_int,
12268 multiple_llvm_types,
12269 slice,
12270 float_array: u8,
12271 i32_array: u8,
12272 i64_array: u8,
12273 };
12274
12275 pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
12276 if (it.zig_index >= it.fn_info.param_types.len) return null;
12277 const ip = &it.pt.zcu.intern_pool;
12278 const ty = it.fn_info.param_types.get(ip)[it.zig_index];
12279 it.byval_attr = false;
12280 return nextInner(it, Type.fromInterned(ty));
12281 }
12282
12283 /// `airCall` uses this instead of `next` so that it can take into account variadic functions.
12284 pub fn nextCall(it: *ParamTypeIterator, fg: *FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
12285 assert(std.meta.eql(it.pt, fg.ng.pt));
12286 const ip = &it.pt.zcu.intern_pool;
12287 if (it.zig_index >= it.fn_info.param_types.len) {
12288 if (it.zig_index >= args.len) {
12289 return null;
12290 } else {
12291 return nextInner(it, fg.typeOf(args[it.zig_index]));
12292 }
12293 } else {
12294 return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index]));
12295 }
12296 }
12297
12298 fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12299 const pt = it.pt;
12300 const zcu = pt.zcu;
12301 const target = zcu.getTarget();
12302
12303 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {
12304 it.zig_index += 1;
12305 return .no_bits;
12306 }
12307 switch (it.fn_info.cc) {
12308 .@"inline" => unreachable,
12309 .auto => {
12310 it.zig_index += 1;
12311 it.llvm_index += 1;
12312 if (ty.isSlice(zcu) or
12313 (ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
12314 {
12315 it.llvm_index += 1;
12316 return .slice;
12317 } else if (isByRef(ty, zcu)) {
12318 return .byref;
12319 } else if (target.cpu.arch.isX86() and
12320 !target.cpu.has(.x86, .evex512) and
12321 ty.totalVectorBits(zcu) >= 512)
12322 {
12323 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
12324 // "512-bit vector arguments require 'evex512' for AVX512"
12325 return .byref;
12326 } else {
12327 return .byval;
12328 }
12329 },
12330 .async => {
12331 @panic("TODO implement async function lowering in the LLVM backend");
12332 },
12333 .x86_64_sysv => return it.nextSystemV(ty),
12334 .x86_64_win => return it.nextWin64(ty),
12335 .x86_stdcall => {
12336 it.zig_index += 1;
12337 it.llvm_index += 1;
12338
12339 if (isScalar(zcu, ty)) {
12340 return .byval;
12341 } else {
12342 it.byval_attr = true;
12343 return .byref;
12344 }
12345 },
12346 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
12347 it.zig_index += 1;
12348 it.llvm_index += 1;
12349 switch (aarch64_c_abi.classifyType(ty, zcu)) {
12350 .memory => return .byref_mut,
12351 .float_array => |len| return Lowering{ .float_array = len },
12352 .byval => return .byval,
12353 .integer => {
12354 it.types_len = 1;
12355 it.types_buffer[0] = .i64;
12356 return .multiple_llvm_types;
12357 },
12358 .double_integer => return Lowering{ .i64_array = 2 },
12359 }
12360 },
12361 .arm_aapcs, .arm_aapcs_vfp => {
12362 it.zig_index += 1;
12363 it.llvm_index += 1;
12364 switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
12365 .memory => {
12366 it.byval_attr = true;
12367 return .byref;
12368 },
12369 .byval => return .byval,
12370 .i32_array => |size| return Lowering{ .i32_array = size },
12371 .i64_array => |size| return Lowering{ .i64_array = size },
12372 }
12373 },
12374 .mips_o32 => {
12375 it.zig_index += 1;
12376 it.llvm_index += 1;
12377 switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
12378 .memory => {
12379 it.byval_attr = true;
12380 return .byref;
12381 },
12382 .byval => return .byval,
12383 .i32_array => |size| return Lowering{ .i32_array = size },
12384 }
12385 },
12386 .riscv64_lp64, .riscv32_ilp32 => {
12387 it.zig_index += 1;
12388 it.llvm_index += 1;
12389 switch (riscv_c_abi.classifyType(ty, zcu)) {
12390 .memory => return .byref_mut,
12391 .byval => return .byval,
12392 .integer => return .abi_sized_int,
12393 .double_integer => return Lowering{ .i64_array = 2 },
12394 .fields => {
12395 it.types_len = 0;
12396 for (0..ty.structFieldCount(zcu)) |field_index| {
12397 const field_ty = ty.fieldType(field_index, zcu);
12398 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
12399 it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty);
12400 it.types_len += 1;
12401 }
12402 it.llvm_index += it.types_len - 1;
12403 return .multiple_llvm_types;
12404 },
12405 }
12406 },
12407 .wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) {
12408 .direct => |scalar_ty| {
12409 if (isScalar(zcu, ty)) {
12410 it.zig_index += 1;
12411 it.llvm_index += 1;
12412 return .byval;
12413 } else {
12414 var types_buffer: [8]Builder.Type = undefined;
12415 types_buffer[0] = try it.object.lowerType(pt, scalar_ty);
12416 it.types_buffer = types_buffer;
12417 it.types_len = 1;
12418 it.llvm_index += 1;
12419 it.zig_index += 1;
12420 return .multiple_llvm_types;
12421 }
12422 },
12423 .indirect => {
12424 it.zig_index += 1;
12425 it.llvm_index += 1;
12426 it.byval_attr = true;
12427 return .byref;
12428 },
12429 },
12430 // TODO investigate other callconvs
12431 else => {
12432 it.zig_index += 1;
12433 it.llvm_index += 1;
12434 return .byval;
12435 },
12436 }
12437 }
12438
12439 fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
12440 const zcu = it.pt.zcu;
12441 switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
12442 .integer => {
12443 if (isScalar(zcu, ty)) {
12444 it.zig_index += 1;
12445 it.llvm_index += 1;
12446 return .byval;
12447 } else {
12448 it.zig_index += 1;
12449 it.llvm_index += 1;
12450 return .abi_sized_int;
12451 }
12452 },
12453 .win_i128 => {
12454 it.zig_index += 1;
12455 it.llvm_index += 1;
12456 return .byref;
12457 },
12458 .memory => {
12459 it.zig_index += 1;
12460 it.llvm_index += 1;
12461 return .byref_mut;
12462 },
12463 .sse => {
12464 it.zig_index += 1;
12465 it.llvm_index += 1;
12466 return .byval;
12467 },
12468 else => unreachable,
12469 }
12470 }
12471
12472 fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12473 const zcu = it.pt.zcu;
12474 const ip = &zcu.intern_pool;
12475 const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg);
12476 if (classes[0] == .memory) {
12477 it.zig_index += 1;
12478 it.llvm_index += 1;
12479 it.byval_attr = true;
12480 return .byref;
12481 }
12482 if (isScalar(zcu, ty)) {
12483 it.zig_index += 1;
12484 it.llvm_index += 1;
12485 return .byval;
12486 }
12487 var types_index: u32 = 0;
12488 var types_buffer: [8]Builder.Type = undefined;
12489 for (classes) |class| {
12490 switch (class) {
12491 .integer => {
12492 types_buffer[types_index] = .i64;
12493 types_index += 1;
12494 },
12495 .sse => {
12496 types_buffer[types_index] = .double;
12497 types_index += 1;
12498 },
12499 .sseup => {
12500 if (types_buffer[types_index - 1] == .double) {
12501 types_buffer[types_index - 1] = .fp128;
12502 } else {
12503 types_buffer[types_index] = .double;
12504 types_index += 1;
12505 }
12506 },
12507 .float => {
12508 types_buffer[types_index] = .float;
12509 types_index += 1;
12510 },
12511 .float_combine => {
12512 types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
12513 types_index += 1;
12514 },
12515 .x87 => {
12516 it.zig_index += 1;
12517 it.llvm_index += 1;
12518 it.byval_attr = true;
12519 return .byref;
12520 },
12521 .x87up => unreachable,
12522 .none => break,
12523 .memory => unreachable, // handled above
12524 .win_i128 => unreachable, // windows only
12525 .integer_per_element => {
12526 @panic("TODO");
12527 },
12528 }
12529 }
12530 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12531 if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12532 assert(first_non_integer orelse classes.len == types_index);
12533 if (types_index == 1) {
12534 it.zig_index += 1;
12535 it.llvm_index += 1;
12536 return .abi_sized_int;
12537 }
12538 if (it.llvm_index + types_index > 6) {
12539 it.zig_index += 1;
12540 it.llvm_index += 1;
12541 it.byval_attr = true;
12542 return .byref;
12543 }
12544 switch (ip.indexToKey(ty.toIntern())) {
12545 .struct_type => {
12546 const struct_type = ip.loadStructType(ty.toIntern());
12547 assert(struct_type.haveLayout(ip));
12548 const size: u64 = struct_type.sizeUnordered(ip);
12549 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12550 if (size % 8 > 0) {
12551 types_buffer[types_index - 1] =
12552 try it.object.builder.intType(@intCast(size % 8 * 8));
12553 }
12554 },
12555 else => {},
12556 }
12557 }
12558 it.types_len = types_index;
12559 it.types_buffer = types_buffer;
12560 it.llvm_index += types_index;
12561 it.zig_index += 1;
12562 return .multiple_llvm_types;
12563 }
12564};
12565
12566fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator {
12567 return .{
12568 .object = object,
12569 .pt = pt,
12570 .fn_info = fn_info,
12571 .zig_index = 0,
12572 .llvm_index = 0,
12573 .types_len = 0,
12574 .types_buffer = undefined,
12575 .byval_attr = false,
12576 };
12577}
12578
12579fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness {
12580 const target = zcu.getTarget();
12581 switch (cc) {
12582 .auto, .@"inline", .async => return null,
12583 else => {},
12584 }
12585 const int_info = switch (ty.zigTypeTag(zcu)) {
12586 .bool => Type.u1.intInfo(zcu),
12587 .int, .@"enum", .error_set => ty.intInfo(zcu),
12588 else => return null,
12589 };
12590 return switch (target.os.tag) {
12591 .driverkit, .ios, .maccatalyst, .macos, .watchos, .tvos, .visionos => switch (int_info.bits) {
12592 0...16 => int_info.signedness,
12593 else => null,
12594 },
12595 else => switch (target.cpu.arch) {
12596 .loongarch64, .riscv64, .riscv64be => switch (int_info.bits) {
12597 0...16 => int_info.signedness,
12598 32 => .signed, // LLVM always signextends 32 bit ints, unsure if bug.
12599 17...31, 33...63 => int_info.signedness,
12600 else => null,
12601 },
12602
12603 .sparc64,
12604 .powerpc64,
12605 .powerpc64le,
12606 .s390x,
12607 => switch (int_info.bits) {
12608 0...63 => int_info.signedness,
12609 else => null,
12610 },
12611
12612 .aarch64,
12613 .aarch64_be,
12614 => null,
12615
12616 else => switch (int_info.bits) {
12617 0...16 => int_info.signedness,
12618 else => null,
12619 },
12620 },
12621 };
12622}
12623
12624/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
12625/// or as an LLVM value.
12626fn isByRef(ty: Type, zcu: *Zcu) bool {
12627 // For tuples and structs, if there are more than this many non-void
12628 // fields, then we make it byref, otherwise byval.
12629 const max_fields_byval = 0;
12630 const ip = &zcu.intern_pool;
12631
12632 switch (ty.zigTypeTag(zcu)) {
12633 .type,
12634 .comptime_int,
12635 .comptime_float,
12636 .enum_literal,
12637 .undefined,
12638 .null,
12639 .@"opaque",
12640 => unreachable,
12641
12642 .noreturn,
12643 .void,
12644 .bool,
12645 .int,
12646 .float,
12647 .pointer,
12648 .error_set,
12649 .@"fn",
12650 .@"enum",
12651 .vector,
12652 .@"anyframe",
12653 => return false,
12654
12655 .array, .frame => return ty.hasRuntimeBits(zcu),
12656 .@"struct" => {
12657 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
12658 .tuple_type => |tuple| {
12659 var count: usize = 0;
12660 for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
12661 if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
12662
12663 count += 1;
12664 if (count > max_fields_byval) return true;
12665 if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
12666 }
12667 return false;
12668 },
12669 .struct_type => ip.loadStructType(ty.toIntern()),
12670 else => unreachable,
12671 };
12672
12673 // Packed structs are represented to LLVM as integers.
12674 if (struct_type.layout == .@"packed") return false;
12675
12676 const field_types = struct_type.field_types.get(ip);
12677 var it = struct_type.iterateRuntimeOrder(ip);
12678 var count: usize = 0;
12679 while (it.next()) |field_index| {
12680 count += 1;
12681 if (count > max_fields_byval) return true;
12682 const field_ty = Type.fromInterned(field_types[field_index]);
12683 if (isByRef(field_ty, zcu)) return true;
12684 }
12685 return false;
12686 },
12687 .@"union" => switch (ty.containerLayout(zcu)) {
12688 .@"packed" => return false,
12689 else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
12690 },
12691 .error_union => {
12692 const payload_ty = ty.errorUnionPayload(zcu);
12693 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
12694 return false;
12695 }
12696 return true;
12697 },
12698 .optional => {
12699 const payload_ty = ty.optionalChild(zcu);
12700 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
12701 return false;
12702 }
12703 if (ty.optionalReprIsPayload(zcu)) {
12704 return false;
12705 }
12706 return true;
12707 },
12708 }
12709}
12710
12711fn isScalar(zcu: *Zcu, ty: Type) bool {
12712 return switch (ty.zigTypeTag(zcu)) {
12713 .void,
12714 .bool,
12715 .noreturn,
12716 .int,
12717 .float,
12718 .pointer,
12719 .optional,
12720 .error_set,
12721 .@"enum",
12722 .@"anyframe",
12723 .vector,
12724 => true,
12725
12726 .@"struct" => ty.containerLayout(zcu) == .@"packed",
12727 .@"union" => ty.containerLayout(zcu) == .@"packed",
12728 else => false,
12729 };
12730}
12731
12732/// This function returns true if we expect LLVM to lower x86_fp80 correctly
12733/// and false if we expect LLVM to crash if it encounters an x86_fp80 type,
12734/// or if it produces miscompilations.
12735fn backendSupportsF80(target: *const std.Target) bool {
12736 return switch (target.cpu.arch) {
12737 .x86, .x86_64 => !target.cpu.has(.x86, .soft_float),
12738 else => false,
12739 };
12740}
12741
12742/// This function returns true if we expect LLVM to lower f16 correctly
12743/// and false if we expect LLVM to crash if it encounters an f16 type,
12744/// or if it produces miscompilations.
12745fn backendSupportsF16(target: *const std.Target) bool {
12746 return switch (target.cpu.arch) {
12747 // https://github.com/llvm/llvm-project/issues/97981
12748 .csky,
12749 // https://github.com/llvm/llvm-project/issues/97981
12750 .powerpc,
12751 .powerpcle,
12752 .powerpc64,
12753 .powerpc64le,
12754 // https://github.com/llvm/llvm-project/issues/97981
12755 .wasm32,
12756 .wasm64,
12757 // https://github.com/llvm/llvm-project/issues/97981
12758 .sparc,
12759 .sparc64,
12760 => false,
12761 .arm,
12762 .armeb,
12763 .thumb,
12764 .thumbeb,
12765 => target.abi.float() == .soft or target.cpu.has(.arm, .fullfp16),
12766 else => true,
12767 };
12768}
12769
12770/// This function returns true if we expect LLVM to lower f128 correctly,
12771/// and false if we expect LLVM to crash if it encounters an f128 type,
12772/// or if it produces miscompilations.
12773fn backendSupportsF128(target: *const std.Target) bool {
12774 return switch (target.cpu.arch) {
12775 // https://github.com/llvm/llvm-project/issues/121122
12776 .amdgcn,
12777 // Test failures all over the place.
12778 .mips64,
12779 .mips64el,
12780 // https://github.com/llvm/llvm-project/issues/41838
12781 .sparc,
12782 => false,
12783 .arm,
12784 .armeb,
12785 .thumb,
12786 .thumbeb,
12787 => target.abi.float() == .soft or target.cpu.has(.arm, .fp_armv8),
12788 else => true,
12789 };
12790}
12791
12792/// LLVM does not support all relevant intrinsics for all targets, so we
12793/// may need to manually generate a compiler-rt call.
12794fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool {
12795 return switch (scalar_ty.toIntern()) {
12796 .f16_type => backendSupportsF16(target),
12797 .f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target),
12798 .f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target),
12799 else => true,
12800 };
12801}
12802
12803/// We need to insert extra padding if LLVM's isn't enough.
12804/// However we don't want to ever call LLVMABIAlignmentOfType or
12805/// LLVMABISizeOfType because these functions will trip assertions
12806/// when using them for self-referential types. So our strategy is
12807/// to use non-packed llvm structs but to emit all padding explicitly.
12808/// We can do this because for all types, Zig ABI alignment >= LLVM ABI
12809/// alignment.
12810const struct_layout_version = 2;
12811
12812// TODO: Restore the non_null field to i1 once
12813// https://github.com/llvm/llvm-project/issues/56585/ is fixed
12814const optional_layout_version = 3;
12815
12816const lt_errors_fn_name = "__zig_lt_errors_len";
12817
12818fn compilerRtIntBits(bits: u16) ?u16 {
12819 inline for (.{ 32, 64, 128 }) |b| {
12820 if (bits <= b) {
12821 return b;
12822 }
12823 }
12824 return null;
12825}
12826
12827fn buildAllocaInner(
12828 wip: *Builder.WipFunction,
12829 llvm_ty: Builder.Type,
12830 alignment: Builder.Alignment,
12831 target: *const std.Target,
12832) Allocator.Error!Builder.Value {
12833 const address_space = llvmAllocaAddressSpace(target);
12834
12835 const alloca = blk: {
12836 const prev_cursor = wip.cursor;
12837 const prev_debug_location = wip.debug_location;
12838 defer {
12839 wip.cursor = prev_cursor;
12840 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
12841 wip.debug_location = prev_debug_location;
12842 }
12843
12844 wip.cursor = .{ .block = .entry };
12845 wip.debug_location = .no_location;
12846 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
12847 };
12848
12849 // The pointer returned from this function should have the generic address space,
12850 // if this isn't the case then cast it to the generic address space.
12851 return wip.conv(.unneeded, alloca, .ptr, "");
12852}
12853
12854fn errUnionPayloadOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12855 const zcu = pt.zcu;
12856 const err_int_ty = try pt.errorIntType();
12857 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));
12858}
12859
12860fn errUnionErrorOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12861 const zcu = pt.zcu;
12862 const err_int_ty = try pt.errorIntType();
12863 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));
12864}
12865
12866/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
12867///
12868/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12869fn constraintAllowsMemory(constraint: []const u8) bool {
12870 // TODO: This implementation is woefully incomplete.
12871 for (constraint) |byte| {
12872 switch (byte) {
12873 '=', '*', ',', '&' => {},
12874 'm', 'o', 'X', 'g' => return true,
12875 else => {},
12876 }
12877 } else return false;
12878}
12879
12880/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
12881///
12882/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12883fn constraintAllowsRegister(constraint: []const u8) bool {
12884 // TODO: This implementation is woefully incomplete.
12885 for (constraint) |byte| {
12886 switch (byte) {
12887 '=', '*', ',', '&' => {},
12888 'm', 'o' => {},
12889 else => return true,
12890 }
12891 } else return false;
12892}
12893
12894pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
12895 switch (arch) {
12896 .aarch64, .aarch64_be => {
12897 llvm.LLVMInitializeAArch64Target();
12898 llvm.LLVMInitializeAArch64TargetInfo();
12899 llvm.LLVMInitializeAArch64TargetMC();
12900 llvm.LLVMInitializeAArch64AsmPrinter();
12901 llvm.LLVMInitializeAArch64AsmParser();
12902 },
12903 .amdgcn => {
12904 llvm.LLVMInitializeAMDGPUTarget();
12905 llvm.LLVMInitializeAMDGPUTargetInfo();
12906 llvm.LLVMInitializeAMDGPUTargetMC();
12907 llvm.LLVMInitializeAMDGPUAsmPrinter();
12908 llvm.LLVMInitializeAMDGPUAsmParser();
12909 },
12910 .thumb, .thumbeb, .arm, .armeb => {
12911 llvm.LLVMInitializeARMTarget();
12912 llvm.LLVMInitializeARMTargetInfo();
12913 llvm.LLVMInitializeARMTargetMC();
12914 llvm.LLVMInitializeARMAsmPrinter();
12915 llvm.LLVMInitializeARMAsmParser();
12916 },
12917 .avr => {
12918 llvm.LLVMInitializeAVRTarget();
12919 llvm.LLVMInitializeAVRTargetInfo();
12920 llvm.LLVMInitializeAVRTargetMC();
12921 llvm.LLVMInitializeAVRAsmPrinter();
12922 llvm.LLVMInitializeAVRAsmParser();
12923 },
12924 .bpfel, .bpfeb => {
12925 llvm.LLVMInitializeBPFTarget();
12926 llvm.LLVMInitializeBPFTargetInfo();
12927 llvm.LLVMInitializeBPFTargetMC();
12928 llvm.LLVMInitializeBPFAsmPrinter();
12929 llvm.LLVMInitializeBPFAsmParser();
12930 },
12931 .hexagon => {
12932 llvm.LLVMInitializeHexagonTarget();
12933 llvm.LLVMInitializeHexagonTargetInfo();
12934 llvm.LLVMInitializeHexagonTargetMC();
12935 llvm.LLVMInitializeHexagonAsmPrinter();
12936 llvm.LLVMInitializeHexagonAsmParser();
12937 },
12938 .lanai => {
12939 llvm.LLVMInitializeLanaiTarget();
12940 llvm.LLVMInitializeLanaiTargetInfo();
12941 llvm.LLVMInitializeLanaiTargetMC();
12942 llvm.LLVMInitializeLanaiAsmPrinter();
12943 llvm.LLVMInitializeLanaiAsmParser();
12944 },
12945 .mips, .mipsel, .mips64, .mips64el => {
12946 llvm.LLVMInitializeMipsTarget();
12947 llvm.LLVMInitializeMipsTargetInfo();
12948 llvm.LLVMInitializeMipsTargetMC();
12949 llvm.LLVMInitializeMipsAsmPrinter();
12950 llvm.LLVMInitializeMipsAsmParser();
12951 },
12952 .msp430 => {
12953 llvm.LLVMInitializeMSP430Target();
12954 llvm.LLVMInitializeMSP430TargetInfo();
12955 llvm.LLVMInitializeMSP430TargetMC();
12956 llvm.LLVMInitializeMSP430AsmPrinter();
12957 llvm.LLVMInitializeMSP430AsmParser();
12958 },
12959 .nvptx, .nvptx64 => {
12960 llvm.LLVMInitializeNVPTXTarget();
12961 llvm.LLVMInitializeNVPTXTargetInfo();
12962 llvm.LLVMInitializeNVPTXTargetMC();
12963 llvm.LLVMInitializeNVPTXAsmPrinter();
12964 // There is no LLVMInitializeNVPTXAsmParser function available.
12965 },
12966 .powerpc, .powerpcle, .powerpc64, .powerpc64le => {
12967 llvm.LLVMInitializePowerPCTarget();
12968 llvm.LLVMInitializePowerPCTargetInfo();
12969 llvm.LLVMInitializePowerPCTargetMC();
12970 llvm.LLVMInitializePowerPCAsmPrinter();
12971 llvm.LLVMInitializePowerPCAsmParser();
12972 },
12973 .riscv32, .riscv32be, .riscv64, .riscv64be => {
12974 llvm.LLVMInitializeRISCVTarget();
12975 llvm.LLVMInitializeRISCVTargetInfo();
12976 llvm.LLVMInitializeRISCVTargetMC();
12977 llvm.LLVMInitializeRISCVAsmPrinter();
12978 llvm.LLVMInitializeRISCVAsmParser();
12979 },
12980 .sparc, .sparc64 => {
12981 llvm.LLVMInitializeSparcTarget();
12982 llvm.LLVMInitializeSparcTargetInfo();
12983 llvm.LLVMInitializeSparcTargetMC();
12984 llvm.LLVMInitializeSparcAsmPrinter();
12985 llvm.LLVMInitializeSparcAsmParser();
12986 },
12987 .s390x => {
12988 llvm.LLVMInitializeSystemZTarget();
12989 llvm.LLVMInitializeSystemZTargetInfo();
12990 llvm.LLVMInitializeSystemZTargetMC();
12991 llvm.LLVMInitializeSystemZAsmPrinter();
12992 llvm.LLVMInitializeSystemZAsmParser();
12993 },
12994 .wasm32, .wasm64 => {
12995 llvm.LLVMInitializeWebAssemblyTarget();
12996 llvm.LLVMInitializeWebAssemblyTargetInfo();
12997 llvm.LLVMInitializeWebAssemblyTargetMC();
12998 llvm.LLVMInitializeWebAssemblyAsmPrinter();
12999 llvm.LLVMInitializeWebAssemblyAsmParser();
13000 },
13001 .x86, .x86_64 => {
13002 llvm.LLVMInitializeX86Target();
13003 llvm.LLVMInitializeX86TargetInfo();
13004 llvm.LLVMInitializeX86TargetMC();
13005 llvm.LLVMInitializeX86AsmPrinter();
13006 llvm.LLVMInitializeX86AsmParser();
13007 },
13008 .xtensa => {
13009 if (build_options.llvm_has_xtensa) {
13010 llvm.LLVMInitializeXtensaTarget();
13011 llvm.LLVMInitializeXtensaTargetInfo();
13012 llvm.LLVMInitializeXtensaTargetMC();
13013 // There is no LLVMInitializeXtensaAsmPrinter function.
13014 llvm.LLVMInitializeXtensaAsmParser();
13015 }
13016 },
13017 .xcore => {
13018 llvm.LLVMInitializeXCoreTarget();
13019 llvm.LLVMInitializeXCoreTargetInfo();
13020 llvm.LLVMInitializeXCoreTargetMC();
13021 llvm.LLVMInitializeXCoreAsmPrinter();
13022 // There is no LLVMInitializeXCoreAsmParser function.
13023 },
13024 .m68k => {
13025 if (build_options.llvm_has_m68k) {
13026 llvm.LLVMInitializeM68kTarget();
13027 llvm.LLVMInitializeM68kTargetInfo();
13028 llvm.LLVMInitializeM68kTargetMC();
13029 llvm.LLVMInitializeM68kAsmPrinter();
13030 llvm.LLVMInitializeM68kAsmParser();
13031 }
13032 },
13033 .csky => {
13034 if (build_options.llvm_has_csky) {
13035 llvm.LLVMInitializeCSKYTarget();
13036 llvm.LLVMInitializeCSKYTargetInfo();
13037 llvm.LLVMInitializeCSKYTargetMC();
13038 // There is no LLVMInitializeCSKYAsmPrinter function.
13039 llvm.LLVMInitializeCSKYAsmParser();
13040 }
13041 },
13042 .ve => {
13043 llvm.LLVMInitializeVETarget();
13044 llvm.LLVMInitializeVETargetInfo();
13045 llvm.LLVMInitializeVETargetMC();
13046 llvm.LLVMInitializeVEAsmPrinter();
13047 llvm.LLVMInitializeVEAsmParser();
13048 },
13049 .arc => {
13050 if (build_options.llvm_has_arc) {
13051 llvm.LLVMInitializeARCTarget();
13052 llvm.LLVMInitializeARCTargetInfo();
13053 llvm.LLVMInitializeARCTargetMC();
13054 llvm.LLVMInitializeARCAsmPrinter();
13055 // There is no LLVMInitializeARCAsmParser function.
13056 }
13057 },
13058 .loongarch32, .loongarch64 => {
13059 llvm.LLVMInitializeLoongArchTarget();
13060 llvm.LLVMInitializeLoongArchTargetInfo();
13061 llvm.LLVMInitializeLoongArchTargetMC();
13062 llvm.LLVMInitializeLoongArchAsmPrinter();
13063 llvm.LLVMInitializeLoongArchAsmParser();
13064 },
13065 .spirv32,
13066 .spirv64,
13067 => {
13068 llvm.LLVMInitializeSPIRVTarget();
13069 llvm.LLVMInitializeSPIRVTargetInfo();
13070 llvm.LLVMInitializeSPIRVTargetMC();
13071 llvm.LLVMInitializeSPIRVAsmPrinter();
13072 },
13073
13074 // LLVM does does not have a backend for these.
13075 .alpha,
13076 .arceb,
13077 .hppa,
13078 .hppa64,
13079 .kalimba,
13080 .kvx,
13081 .microblaze,
13082 .microblazeel,
13083 .or1k,
13084 .propeller,
13085 .sh,
13086 .sheb,
13087 .x86_16,
13088 .xtensaeb,
13089 => unreachable,
13090 }
13091}
13092
13093fn minIntConst(b: *Builder, min_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant {
13094 const info = min_ty.intInfo(zcu);
13095 if (info.signedness == .unsigned or info.bits == 0) {
13096 return b.intConst(as_ty, 0);
13097 }
13098 if (std.math.cast(u6, info.bits - 1)) |shift| {
13099 const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
13100 return b.intConst(as_ty, min_val);
13101 }
13102 var res: std.math.big.int.Managed = try .init(zcu.gpa);
13103 defer res.deinit();
13104 try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
13105 return b.bigIntConst(as_ty, res.toConst());
13106}
13107
13108fn maxIntConst(b: *Builder, max_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant {
13109 const info = max_ty.intInfo(zcu);
13110 switch (info.bits) {
13111 0 => return b.intConst(as_ty, 0),
13112 1 => switch (info.signedness) {
13113 .signed => return b.intConst(as_ty, 0),
13114 .unsigned => return b.intConst(as_ty, 1),
13115 },
13116 else => {},
13117 }
13118 const unsigned_bits = switch (info.signedness) {
13119 .unsigned => info.bits,
13120 .signed => info.bits - 1,
13121 };
13122 if (std.math.cast(u6, unsigned_bits)) |shift| {
13123 const max_val: u64 = (@as(u64, 1) << shift) - 1;
13124 return b.intConst(as_ty, max_val);
13125 }
13126 var res: std.math.big.int.Managed = try .init(zcu.gpa);
13127 defer res.deinit();
13128 try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
13129 return b.bigIntConst(as_ty, res.toConst());
13130}
13131
13132/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
13133fn appendConstraints(
13134 gpa: Allocator,
13135 llvm_constraints: *std.ArrayList(u8),
13136 zig_name: []const u8,
13137 target: *const std.Target,
13138) error{OutOfMemory}!usize {
13139 switch (target.cpu.arch) {
13140 .mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) |llvm_tag| {
13141 const llvm_name = @tagName(llvm_tag);
13142 try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
13143 llvm_constraints.appendSliceAssumeCapacity("~{");
13144 llvm_constraints.appendSliceAssumeCapacity(llvm_name);
13145 llvm_constraints.appendSliceAssumeCapacity("},");
13146 return 1;
13147 },
13148 else => {},
13149 }
13150
13151 try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
13152 llvm_constraints.appendSliceAssumeCapacity("~{");
13153 llvm_constraints.appendSliceAssumeCapacity(zig_name);
13154 llvm_constraints.appendSliceAssumeCapacity("},");
13155 return 1;
13156}
13157
13158const mips_clobber_overrides = std.StaticStringMap(enum {
13159 @"$msair",
13160 @"$msacsr",
13161 @"$msaaccess",
13162 @"$msasave",
13163 @"$msamodify",
13164 @"$msarequest",
13165 @"$msamap",
13166 @"$msaunmap",
13167 @"$f0",
13168 @"$f1",
13169 @"$f2",
13170 @"$f3",
13171 @"$f4",
13172 @"$f5",
13173 @"$f6",
13174 @"$f7",
13175 @"$f8",
13176 @"$f9",
13177 @"$f10",
13178 @"$f11",
13179 @"$f12",
13180 @"$f13",
13181 @"$f14",
13182 @"$f15",
13183 @"$f16",
13184 @"$f17",
13185 @"$f18",
13186 @"$f19",
13187 @"$f20",
13188 @"$f21",
13189 @"$f22",
13190 @"$f23",
13191 @"$f24",
13192 @"$f25",
13193 @"$f26",
13194 @"$f27",
13195 @"$f28",
13196 @"$f29",
13197 @"$f30",
13198 @"$f31",
13199 @"$fcc0",
13200 @"$fcc1",
13201 @"$fcc2",
13202 @"$fcc3",
13203 @"$fcc4",
13204 @"$fcc5",
13205 @"$fcc6",
13206 @"$fcc7",
13207 @"$w0",
13208 @"$w1",
13209 @"$w2",
13210 @"$w3",
13211 @"$w4",
13212 @"$w5",
13213 @"$w6",
13214 @"$w7",
13215 @"$w8",
13216 @"$w9",
13217 @"$w10",
13218 @"$w11",
13219 @"$w12",
13220 @"$w13",
13221 @"$w14",
13222 @"$w15",
13223 @"$w16",
13224 @"$w17",
13225 @"$w18",
13226 @"$w19",
13227 @"$w20",
13228 @"$w21",
13229 @"$w22",
13230 @"$w23",
13231 @"$w24",
13232 @"$w25",
13233 @"$w26",
13234 @"$w27",
13235 @"$w28",
13236 @"$w29",
13237 @"$w30",
13238 @"$w31",
13239 @"$0",
13240 @"$1",
13241 @"$2",
13242 @"$3",
13243 @"$4",
13244 @"$5",
13245 @"$6",
13246 @"$7",
13247 @"$8",
13248 @"$9",
13249 @"$10",
13250 @"$11",
13251 @"$12",
13252 @"$13",
13253 @"$14",
13254 @"$15",
13255 @"$16",
13256 @"$17",
13257 @"$18",
13258 @"$19",
13259 @"$20",
13260 @"$21",
13261 @"$22",
13262 @"$23",
13263 @"$24",
13264 @"$25",
13265 @"$26",
13266 @"$27",
13267 @"$28",
13268 @"$29",
13269 @"$30",
13270 @"$31",
13271}).initComptime(.{
13272 .{ "msa_ir", .@"$msair" },
13273 .{ "msa_csr", .@"$msacsr" },
13274 .{ "msa_access", .@"$msaaccess" },
13275 .{ "msa_save", .@"$msasave" },
13276 .{ "msa_modify", .@"$msamodify" },
13277 .{ "msa_request", .@"$msarequest" },
13278 .{ "msa_map", .@"$msamap" },
13279 .{ "msa_unmap", .@"$msaunmap" },
13280 .{ "f0", .@"$f0" },
13281 .{ "f1", .@"$f1" },
13282 .{ "f2", .@"$f2" },
13283 .{ "f3", .@"$f3" },
13284 .{ "f4", .@"$f4" },
13285 .{ "f5", .@"$f5" },
13286 .{ "f6", .@"$f6" },
13287 .{ "f7", .@"$f7" },
13288 .{ "f8", .@"$f8" },
13289 .{ "f9", .@"$f9" },
13290 .{ "f10", .@"$f10" },
13291 .{ "f11", .@"$f11" },
13292 .{ "f12", .@"$f12" },
13293 .{ "f13", .@"$f13" },
13294 .{ "f14", .@"$f14" },
13295 .{ "f15", .@"$f15" },
13296 .{ "f16", .@"$f16" },
13297 .{ "f17", .@"$f17" },
13298 .{ "f18", .@"$f18" },
13299 .{ "f19", .@"$f19" },
13300 .{ "f20", .@"$f20" },
13301 .{ "f21", .@"$f21" },
13302 .{ "f22", .@"$f22" },
13303 .{ "f23", .@"$f23" },
13304 .{ "f24", .@"$f24" },
13305 .{ "f25", .@"$f25" },
13306 .{ "f26", .@"$f26" },
13307 .{ "f27", .@"$f27" },
13308 .{ "f28", .@"$f28" },
13309 .{ "f29", .@"$f29" },
13310 .{ "f30", .@"$f30" },
13311 .{ "f31", .@"$f31" },
13312 .{ "fcc0", .@"$fcc0" },
13313 .{ "fcc1", .@"$fcc1" },
13314 .{ "fcc2", .@"$fcc2" },
13315 .{ "fcc3", .@"$fcc3" },
13316 .{ "fcc4", .@"$fcc4" },
13317 .{ "fcc5", .@"$fcc5" },
13318 .{ "fcc6", .@"$fcc6" },
13319 .{ "fcc7", .@"$fcc7" },
13320 .{ "w0", .@"$w0" },
13321 .{ "w1", .@"$w1" },
13322 .{ "w2", .@"$w2" },
13323 .{ "w3", .@"$w3" },
13324 .{ "w4", .@"$w4" },
13325 .{ "w5", .@"$w5" },
13326 .{ "w6", .@"$w6" },
13327 .{ "w7", .@"$w7" },
13328 .{ "w8", .@"$w8" },
13329 .{ "w9", .@"$w9" },
13330 .{ "w10", .@"$w10" },
13331 .{ "w11", .@"$w11" },
13332 .{ "w12", .@"$w12" },
13333 .{ "w13", .@"$w13" },
13334 .{ "w14", .@"$w14" },
13335 .{ "w15", .@"$w15" },
13336 .{ "w16", .@"$w16" },
13337 .{ "w17", .@"$w17" },
13338 .{ "w18", .@"$w18" },
13339 .{ "w19", .@"$w19" },
13340 .{ "w20", .@"$w20" },
13341 .{ "w21", .@"$w21" },
13342 .{ "w22", .@"$w22" },
13343 .{ "w23", .@"$w23" },
13344 .{ "w24", .@"$w24" },
13345 .{ "w25", .@"$w25" },
13346 .{ "w26", .@"$w26" },
13347 .{ "w27", .@"$w27" },
13348 .{ "w28", .@"$w28" },
13349 .{ "w29", .@"$w29" },
13350 .{ "w30", .@"$w30" },
13351 .{ "w31", .@"$w31" },
13352 .{ "r0", .@"$0" },
13353 .{ "r1", .@"$1" },
13354 .{ "r2", .@"$2" },
13355 .{ "r3", .@"$3" },
13356 .{ "r4", .@"$4" },
13357 .{ "r5", .@"$5" },
13358 .{ "r6", .@"$6" },
13359 .{ "r7", .@"$7" },
13360 .{ "r8", .@"$8" },
13361 .{ "r9", .@"$9" },
13362 .{ "r10", .@"$10" },
13363 .{ "r11", .@"$11" },
13364 .{ "r12", .@"$12" },
13365 .{ "r13", .@"$13" },
13366 .{ "r14", .@"$14" },
13367 .{ "r15", .@"$15" },
13368 .{ "r16", .@"$16" },
13369 .{ "r17", .@"$17" },
13370 .{ "r18", .@"$18" },
13371 .{ "r19", .@"$19" },
13372 .{ "r20", .@"$20" },
13373 .{ "r21", .@"$21" },
13374 .{ "r22", .@"$22" },
13375 .{ "r23", .@"$23" },
13376 .{ "r24", .@"$24" },
13377 .{ "r25", .@"$25" },
13378 .{ "r26", .@"$26" },
13379 .{ "r27", .@"$27" },
13380 .{ "r28", .@"$28" },
13381 .{ "r29", .@"$29" },
13382 .{ "r30", .@"$30" },
13383 .{ "r31", .@"$31" },
13384});