master
1const std = @import("std");
2const builtin = @import("builtin");
3const build_options = @import("build_options");
4
5const mem = std.mem;
6const math = std.math;
7const assert = std.debug.assert;
8const Allocator = mem.Allocator;
9
10const Air = @import("../../Air.zig");
11const Mir = @import("Mir.zig");
12const Emit = @import("Emit.zig");
13const Type = @import("../../Type.zig");
14const Value = @import("../../Value.zig");
15const link = @import("../../link.zig");
16const Zcu = @import("../../Zcu.zig");
17const Package = @import("../../Package.zig");
18const InternPool = @import("../../InternPool.zig");
19const Compilation = @import("../../Compilation.zig");
20const target_util = @import("../../target.zig");
21const trace = @import("../../tracy.zig").trace;
22const codegen = @import("../../codegen.zig");
23
24const ErrorMsg = Zcu.ErrorMsg;
25const Target = std.Target;
26
27const log = std.log.scoped(.riscv_codegen);
28const tracking_log = std.log.scoped(.tracking);
29const verbose_tracking_log = std.log.scoped(.verbose_tracking);
30const wip_mir_log = std.log.scoped(.wip_mir);
31const Alignment = InternPool.Alignment;
32
33const CodeGenError = codegen.CodeGenError;
34
35const bits = @import("bits.zig");
36const abi = @import("abi.zig");
37const Lower = @import("Lower.zig");
38const mnem_import = @import("mnem.zig");
39const Mnemonic = mnem_import.Mnemonic;
40const Pseudo = mnem_import.Pseudo;
41const encoding = @import("encoding.zig");
42
43const Register = bits.Register;
44const CSR = bits.CSR;
45const Immediate = bits.Immediate;
46const Memory = bits.Memory;
47const FrameIndex = bits.FrameIndex;
48const RegisterManager = abi.RegisterManager;
49const RegisterLock = RegisterManager.RegisterLock;
50const Instruction = encoding.Instruction;
51
52const InnerError = CodeGenError || error{OutOfRegisters};
53
54pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
55 return comptime &.initMany(&.{
56 .expand_intcast_safe,
57 .expand_int_from_float_safe,
58 .expand_int_from_float_optimized_safe,
59 .expand_add_safe,
60 .expand_sub_safe,
61 .expand_mul_safe,
62 });
63}
64
65pt: Zcu.PerThread,
66air: Air,
67liveness: Air.Liveness,
68bin_file: *link.File,
69gpa: Allocator,
70
71mod: *Package.Module,
72target: *const std.Target,
73args: []MCValue,
74ret_mcv: InstTracking,
75func_index: InternPool.Index,
76fn_type: Type,
77arg_index: usize,
78src_loc: Zcu.LazySrcLoc,
79
80mir_instructions: std.MultiArrayList(Mir.Inst) = .{},
81
82owner: Owner,
83
84/// Byte offset within the source file of the ending curly.
85end_di_line: u32,
86end_di_column: u32,
87
88scope_generation: u32,
89
90/// The value is an offset into the `Function` `code` from the beginning.
91/// To perform the reloc, write 32-bit signed little-endian integer
92/// which is a relative jump, based on the address following the reloc.
93exitlude_jump_relocs: std.ArrayList(usize) = .empty,
94
95reused_operands: std.StaticBitSet(Air.Liveness.bpi - 1) = undefined,
96
97/// Whenever there is a runtime branch, we push a Branch onto this stack,
98/// and pop it off when the runtime branch joins. This provides an "overlay"
99/// of the table of mappings from instructions to `MCValue` from within the branch.
100/// This way we can modify the `MCValue` for an instruction in different ways
101/// within different branches. Special consideration is needed when a branch
102/// joins with its parent, to make sure all instructions have the same MCValue
103/// across each runtime branch upon joining.
104branch_stack: *std.array_list.Managed(Branch),
105
106// Currently set vector properties, null means they haven't been set yet in the function.
107avl: ?u64,
108vtype: ?bits.VType,
109
110// Key is the block instruction
111blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .empty,
112register_manager: RegisterManager = .{},
113
114const_tracking: ConstTrackingMap = .{},
115inst_tracking: InstTrackingMap = .{},
116
117frame_allocs: std.MultiArrayList(FrameAlloc) = .{},
118free_frame_indices: std.AutoArrayHashMapUnmanaged(FrameIndex, void) = .empty,
119frame_locs: std.MultiArrayList(Mir.FrameLoc) = .{},
120
121loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
122 /// The state to restore before branching.
123 state: State,
124 /// The branch target.
125 jmp_target: Mir.Inst.Index,
126}) = .{},
127
128/// Debug field, used to find bugs in the compiler.
129air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init,
130
131const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
132
133const SymbolOffset = struct { sym: u32, off: i32 = 0 };
134const RegisterOffset = struct { reg: Register, off: i32 = 0 };
135pub const FrameAddr = struct { index: FrameIndex, off: i32 = 0 };
136
137const Owner = union(enum) {
138 nav_index: InternPool.Nav.Index,
139 lazy_sym: link.File.LazySymbol,
140
141 fn getSymbolIndex(owner: Owner, func: *Func) !u32 {
142 const pt = func.pt;
143 switch (owner) {
144 .nav_index => |nav_index| {
145 const elf_file = func.bin_file.cast(.elf).?;
146 return elf_file.zigObjectPtr().?.getOrCreateMetadataForNav(pt.zcu, nav_index);
147 },
148 .lazy_sym => |lazy_sym| {
149 const elf_file = func.bin_file.cast(.elf).?;
150 return elf_file.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err|
151 func.fail("{s} creating lazy symbol", .{@errorName(err)});
152 },
153 }
154 }
155};
156
157const MCValue = union(enum) {
158 /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc.
159 /// TODO Look into deleting this tag and using `dead` instead, since every use
160 /// of MCValue.none should be instead looking at the type and noticing it is 0 bits.
161 none,
162 /// Control flow will not allow this value to be observed.
163 unreach,
164 /// No more references to this value remain.
165 /// The payload is the value of scope_generation at the point where the death occurred
166 dead: u32,
167 /// The value is undefined. Contains a symbol index to an undefined constant. Null means
168 /// set the undefined value via immediate instead of a load.
169 undef: ?u32,
170 /// A pointer-sized integer that fits in a register.
171 /// If the type is a pointer, this is the pointer address in virtual address space.
172 immediate: u64,
173 /// The value doesn't exist in memory yet.
174 load_symbol: SymbolOffset,
175 /// The address of the memory location not-yet-allocated by the linker.
176 lea_symbol: SymbolOffset,
177 /// The value is in a target-specific register.
178 register: Register,
179 /// The value is split across two registers
180 register_pair: [2]Register,
181 /// The value is in memory at a hard-coded address.
182 /// If the type is a pointer, it means the pointer address is at this memory location.
183 memory: u64,
184 /// The value stored at an offset from a frame index
185 /// Payload is a frame address.
186 load_frame: FrameAddr,
187 /// The address of an offset from a frame index
188 /// Payload is a frame address.
189 lea_frame: FrameAddr,
190 air_ref: Air.Inst.Ref,
191 /// The value is in memory at a constant offset from the address in a register.
192 indirect: RegisterOffset,
193 /// The value is a constant offset from the value in a register.
194 register_offset: RegisterOffset,
195 /// This indicates that we have already allocated a frame index for this instruction,
196 /// but it has not been spilled there yet in the current control flow.
197 /// Payload is a frame index.
198 reserved_frame: FrameIndex,
199
200 fn isMemory(mcv: MCValue) bool {
201 return switch (mcv) {
202 .memory, .indirect, .load_frame => true,
203 else => false,
204 };
205 }
206
207 fn isImmediate(mcv: MCValue) bool {
208 return switch (mcv) {
209 .immediate => true,
210 else => false,
211 };
212 }
213
214 fn isRegister(mcv: MCValue) bool {
215 return switch (mcv) {
216 .register => true,
217 .register_offset => |reg_off| return reg_off.off == 0,
218 else => false,
219 };
220 }
221
222 fn isMutable(mcv: MCValue) bool {
223 return switch (mcv) {
224 .none => unreachable,
225 .unreach => unreachable,
226 .dead => unreachable,
227
228 .immediate,
229 .memory,
230 .lea_frame,
231 .undef,
232 .lea_symbol,
233 .air_ref,
234 .reserved_frame,
235 => false,
236
237 .register,
238 .register_pair,
239 .register_offset,
240 .load_symbol,
241 .indirect,
242 => true,
243
244 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
245 };
246 }
247
248 fn address(mcv: MCValue) MCValue {
249 return switch (mcv) {
250 .none,
251 .unreach,
252 .dead,
253 .immediate,
254 .lea_frame,
255 .register_offset,
256 .register_pair,
257 .register,
258 .undef,
259 .air_ref,
260 .lea_symbol,
261 .reserved_frame,
262 => unreachable, // not in memory
263
264 .load_symbol => |sym_off| .{ .lea_symbol = sym_off },
265 .memory => |addr| .{ .immediate = addr },
266 .load_frame => |off| .{ .lea_frame = off },
267 .indirect => |reg_off| switch (reg_off.off) {
268 0 => .{ .register = reg_off.reg },
269 else => .{ .register_offset = reg_off },
270 },
271 };
272 }
273
274 fn deref(mcv: MCValue) MCValue {
275 return switch (mcv) {
276 .none,
277 .unreach,
278 .dead,
279 .memory,
280 .indirect,
281 .undef,
282 .air_ref,
283 .register_pair,
284 .load_frame,
285 .load_symbol,
286 .reserved_frame,
287 => unreachable, // not a pointer
288
289 .immediate => |addr| .{ .memory = addr },
290 .register => |reg| .{ .indirect = .{ .reg = reg } },
291 .register_offset => |reg_off| .{ .indirect = reg_off },
292 .lea_frame => |off| .{ .load_frame = off },
293 .lea_symbol => |sym_off| .{ .load_symbol = sym_off },
294 };
295 }
296
297 fn offset(mcv: MCValue, off: i32) MCValue {
298 return switch (mcv) {
299 .none,
300 .unreach,
301 .dead,
302 .undef,
303 .air_ref,
304 .reserved_frame,
305 => unreachable, // not valid
306 .register_pair,
307 .memory,
308 .indirect,
309 .load_symbol,
310 .lea_symbol,
311 => switch (off) {
312 0 => mcv,
313 else => unreachable,
314 },
315 .load_frame => |frame| .{ .load_frame = .{ .index = frame.index, .off = frame.off + off } },
316 .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) },
317 .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } },
318 .register_offset => |reg_off| .{ .register_offset = .{ .reg = reg_off.reg, .off = reg_off.off + off } },
319 .lea_frame => |frame_addr| .{
320 .lea_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
321 },
322 };
323 }
324
325 fn getReg(mcv: MCValue) ?Register {
326 return switch (mcv) {
327 .register => |reg| reg,
328 .register_offset, .indirect => |ro| ro.reg,
329 else => null,
330 };
331 }
332
333 fn getRegs(mcv: *const MCValue) []const Register {
334 return switch (mcv.*) {
335 .register => |*reg| @as(*const [1]Register, reg),
336 .register_pair => |*regs| regs,
337 .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg),
338 else => &.{},
339 };
340 }
341};
342
343const Branch = struct {
344 inst_table: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, MCValue) = .empty,
345
346 fn deinit(func: *Branch, gpa: Allocator) void {
347 func.inst_table.deinit(gpa);
348 func.* = undefined;
349 }
350};
351
352const InstTrackingMap = std.AutoArrayHashMapUnmanaged(Air.Inst.Index, InstTracking);
353const ConstTrackingMap = std.AutoArrayHashMapUnmanaged(InternPool.Index, InstTracking);
354
355const InstTracking = struct {
356 long: MCValue,
357 short: MCValue,
358
359 fn init(result: MCValue) InstTracking {
360 return .{ .long = switch (result) {
361 .none,
362 .unreach,
363 .undef,
364 .immediate,
365 .memory,
366 .load_frame,
367 .lea_frame,
368 .load_symbol,
369 .lea_symbol,
370 => result,
371 .dead,
372 .reserved_frame,
373 .air_ref,
374 => unreachable,
375 .register,
376 .register_pair,
377 .register_offset,
378 .indirect,
379 => .none,
380 }, .short = result };
381 }
382
383 fn getReg(inst_tracking: InstTracking) ?Register {
384 return inst_tracking.short.getReg();
385 }
386
387 fn getRegs(inst_tracking: *const InstTracking) []const Register {
388 return inst_tracking.short.getRegs();
389 }
390
391 fn spill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
392 if (std.meta.eql(inst_tracking.long, inst_tracking.short)) return; // Already spilled
393 // Allocate or reuse frame index
394 switch (inst_tracking.long) {
395 .none => inst_tracking.long = try function.allocRegOrMem(
396 function.typeOfIndex(inst),
397 inst,
398 false,
399 ),
400 .load_frame => {},
401 .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } },
402 else => unreachable,
403 }
404 tracking_log.debug("spill %{d} from {} to {}", .{ inst, inst_tracking.short, inst_tracking.long });
405 try function.genCopy(function.typeOfIndex(inst), inst_tracking.long, inst_tracking.short);
406 }
407
408 fn reuseFrame(inst_tracking: *InstTracking) void {
409 switch (inst_tracking.long) {
410 .reserved_frame => |index| inst_tracking.long = .{ .load_frame = .{ .index = index } },
411 else => {},
412 }
413 inst_tracking.short = switch (inst_tracking.long) {
414 .none,
415 .unreach,
416 .undef,
417 .immediate,
418 .memory,
419 .load_frame,
420 .lea_frame,
421 .load_symbol,
422 .lea_symbol,
423 => inst_tracking.long,
424 .dead,
425 .register,
426 .register_pair,
427 .register_offset,
428 .indirect,
429 .reserved_frame,
430 .air_ref,
431 => unreachable,
432 };
433 }
434
435 fn trackSpill(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
436 try function.freeValue(inst_tracking.short);
437 inst_tracking.reuseFrame();
438 tracking_log.debug("%{d} => {f} (spilled)", .{ inst, inst_tracking.* });
439 }
440
441 fn verifyMaterialize(inst_tracking: InstTracking, target: InstTracking) void {
442 switch (inst_tracking.long) {
443 .none,
444 .unreach,
445 .undef,
446 .immediate,
447 .memory,
448 .lea_frame,
449 .load_symbol,
450 .lea_symbol,
451 => assert(std.meta.eql(inst_tracking.long, target.long)),
452 .load_frame,
453 .reserved_frame,
454 => switch (target.long) {
455 .none,
456 .load_frame,
457 .reserved_frame,
458 => {},
459 else => unreachable,
460 },
461 .dead,
462 .register,
463 .register_pair,
464 .register_offset,
465 .indirect,
466 .air_ref,
467 => unreachable,
468 }
469 }
470
471 fn materialize(
472 inst_tracking: *InstTracking,
473 function: *Func,
474 inst: Air.Inst.Index,
475 target: InstTracking,
476 ) !void {
477 inst_tracking.verifyMaterialize(target);
478 try inst_tracking.materializeUnsafe(function, inst, target);
479 }
480
481 fn materializeUnsafe(
482 inst_tracking: InstTracking,
483 function: *Func,
484 inst: Air.Inst.Index,
485 target: InstTracking,
486 ) !void {
487 const ty = function.typeOfIndex(inst);
488 if ((inst_tracking.long == .none or inst_tracking.long == .reserved_frame) and target.long == .load_frame)
489 try function.genCopy(ty, target.long, inst_tracking.short);
490 try function.genCopy(ty, target.short, inst_tracking.short);
491 }
492
493 fn trackMaterialize(inst_tracking: *InstTracking, inst: Air.Inst.Index, target: InstTracking) void {
494 inst_tracking.verifyMaterialize(target);
495 // Don't clobber reserved frame indices
496 inst_tracking.long = if (target.long == .none) switch (inst_tracking.long) {
497 .load_frame => |addr| .{ .reserved_frame = addr.index },
498 .reserved_frame => inst_tracking.long,
499 else => target.long,
500 } else target.long;
501 inst_tracking.short = target.short;
502 tracking_log.debug("%{d} => {f} (materialize)", .{ inst, inst_tracking.* });
503 }
504
505 fn resurrect(inst_tracking: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void {
506 switch (inst_tracking.short) {
507 .dead => |die_generation| if (die_generation >= scope_generation) {
508 inst_tracking.reuseFrame();
509 tracking_log.debug("%{d} => {f} (resurrect)", .{ inst, inst_tracking.* });
510 },
511 else => {},
512 }
513 }
514
515 fn die(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) !void {
516 if (inst_tracking.short == .dead) return;
517 try function.freeValue(inst_tracking.short);
518 inst_tracking.short = .{ .dead = function.scope_generation };
519 tracking_log.debug("%{d} => {f} (death)", .{ inst, inst_tracking.* });
520 }
521
522 fn reuse(
523 inst_tracking: *InstTracking,
524 function: *Func,
525 new_inst: ?Air.Inst.Index,
526 old_inst: Air.Inst.Index,
527 ) void {
528 inst_tracking.short = .{ .dead = function.scope_generation };
529 if (new_inst) |inst|
530 tracking_log.debug("%{d} => {f} (reuse %{d})", .{ inst, inst_tracking.*, old_inst })
531 else
532 tracking_log.debug("tmp => {f} (reuse %{d})", .{ inst_tracking.*, old_inst });
533 }
534
535 fn liveOut(inst_tracking: *InstTracking, function: *Func, inst: Air.Inst.Index) void {
536 for (inst_tracking.getRegs()) |reg| {
537 if (function.register_manager.isRegFree(reg)) {
538 tracking_log.debug("%{d} => {f} (live-out)", .{ inst, inst_tracking.* });
539 continue;
540 }
541
542 const index = RegisterManager.indexOfRegIntoTracked(reg).?;
543 const tracked_inst = function.register_manager.registers[index];
544 const tracking = function.getResolvedInstValue(tracked_inst);
545
546 // Disable death.
547 var found_reg = false;
548 var remaining_reg: Register = .zero;
549 for (tracking.getRegs()) |tracked_reg| if (tracked_reg.id() == reg.id()) {
550 assert(!found_reg);
551 found_reg = true;
552 } else {
553 assert(remaining_reg == .zero);
554 remaining_reg = tracked_reg;
555 };
556 assert(found_reg);
557 tracking.short = switch (remaining_reg) {
558 .zero => .{ .dead = function.scope_generation },
559 else => .{ .register = remaining_reg },
560 };
561
562 // Perform side-effects of freeValue manually.
563 function.register_manager.freeReg(reg);
564
565 tracking_log.debug("%{d} => {f} (live-out %{d})", .{ inst, inst_tracking.*, tracked_inst });
566 }
567 }
568
569 pub fn format(inst_tracking: InstTracking, writer: *std.Io.Writer) std.Io.Writer.Error!void {
570 if (!std.meta.eql(inst_tracking.long, inst_tracking.short)) try writer.print("|{}| ", .{inst_tracking.long});
571 try writer.print("{}", .{inst_tracking.short});
572 }
573};
574
575const FrameAlloc = struct {
576 abi_size: u31,
577 spill_pad: u3,
578 abi_align: Alignment,
579 ref_count: u16,
580
581 fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
582 return .{
583 .abi_size = @intCast(alloc_abi.size),
584 .spill_pad = alloc_abi.pad,
585 .abi_align = alloc_abi.alignment,
586 .ref_count = 0,
587 };
588 }
589 fn initType(ty: Type, zcu: *Zcu) FrameAlloc {
590 return init(.{
591 .size = ty.abiSize(zcu),
592 .alignment = ty.abiAlignment(zcu),
593 });
594 }
595 fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc {
596 const abi_size = ty.abiSize(zcu);
597 const spill_size = if (abi_size < 8)
598 math.ceilPowerOfTwoAssert(u64, abi_size)
599 else
600 std.mem.alignForward(u64, abi_size, 8);
601 return init(.{
602 .size = spill_size,
603 .pad = @intCast(spill_size - abi_size),
604 .alignment = ty.abiAlignment(zcu).maxStrict(
605 Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
606 ),
607 });
608 }
609};
610
611const BlockData = struct {
612 relocs: std.ArrayList(Mir.Inst.Index) = .empty,
613 state: State,
614
615 fn deinit(bd: *BlockData, gpa: Allocator) void {
616 bd.relocs.deinit(gpa);
617 bd.* = undefined;
618 }
619};
620
621const State = struct {
622 registers: RegisterManager.TrackedRegisters,
623 reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking,
624 free_registers: RegisterManager.RegisterBitSet,
625 inst_tracking_len: u32,
626 scope_generation: u32,
627};
628
629fn initRetroactiveState(func: *Func) State {
630 var state: State = undefined;
631 state.inst_tracking_len = @intCast(func.inst_tracking.count());
632 state.scope_generation = func.scope_generation;
633 return state;
634}
635
636fn saveRetroactiveState(func: *Func, state: *State) !void {
637 const free_registers = func.register_manager.free_registers;
638 var it = free_registers.iterator(.{ .kind = .unset });
639 while (it.next()) |index| {
640 const tracked_inst = func.register_manager.registers[index];
641 state.registers[index] = tracked_inst;
642 state.reg_tracking[index] = func.inst_tracking.get(tracked_inst).?;
643 }
644 state.free_registers = free_registers;
645}
646
647fn saveState(func: *Func) !State {
648 var state = func.initRetroactiveState();
649 try func.saveRetroactiveState(&state);
650 return state;
651}
652
653fn restoreState(func: *Func, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
654 emit_instructions: bool,
655 update_tracking: bool,
656 resurrect: bool,
657 close_scope: bool,
658}) !void {
659 if (opts.close_scope) {
660 for (
661 func.inst_tracking.keys()[state.inst_tracking_len..],
662 func.inst_tracking.values()[state.inst_tracking_len..],
663 ) |inst, *tracking| try tracking.die(func, inst);
664 func.inst_tracking.shrinkRetainingCapacity(state.inst_tracking_len);
665 }
666
667 if (opts.resurrect) for (
668 func.inst_tracking.keys()[0..state.inst_tracking_len],
669 func.inst_tracking.values()[0..state.inst_tracking_len],
670 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
671 for (deaths) |death| try func.processDeath(death);
672
673 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).array.len]RegisterLock;
674 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
675 if (opts.update_tracking) {} else std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa);
676
677 var reg_locks = if (opts.update_tracking) {} else try std.array_list.Managed(RegisterLock).initCapacity(
678 stack.get(),
679 @typeInfo(ExpectedContents).array.len,
680 );
681 defer if (!opts.update_tracking) {
682 for (reg_locks.items) |lock| func.register_manager.unlockReg(lock);
683 reg_locks.deinit();
684 };
685
686 for (0..state.registers.len) |index| {
687 const current_maybe_inst = if (func.register_manager.free_registers.isSet(index))
688 null
689 else
690 func.register_manager.registers[index];
691 const target_maybe_inst = if (state.free_registers.isSet(index))
692 null
693 else
694 state.registers[index];
695 if (std.debug.runtime_safety) if (target_maybe_inst) |target_inst|
696 assert(func.inst_tracking.getIndex(target_inst).? < state.inst_tracking_len);
697 if (opts.emit_instructions) {
698 if (current_maybe_inst) |current_inst| {
699 try func.inst_tracking.getPtr(current_inst).?.spill(func, current_inst);
700 }
701 if (target_maybe_inst) |target_inst| {
702 const target_tracking = func.inst_tracking.getPtr(target_inst).?;
703 try target_tracking.materialize(func, target_inst, state.reg_tracking[index]);
704 }
705 }
706 if (opts.update_tracking) {
707 if (current_maybe_inst) |current_inst| {
708 try func.inst_tracking.getPtr(current_inst).?.trackSpill(func, current_inst);
709 }
710 blk: {
711 const inst = target_maybe_inst orelse break :blk;
712 const reg = RegisterManager.regAtTrackedIndex(@intCast(index));
713 func.register_manager.freeReg(reg);
714 func.register_manager.getRegAssumeFree(reg, inst);
715 }
716 if (target_maybe_inst) |target_inst| {
717 func.inst_tracking.getPtr(target_inst).?.trackMaterialize(
718 target_inst,
719 state.reg_tracking[index],
720 );
721 }
722 } else if (target_maybe_inst) |_|
723 try reg_locks.append(func.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
724 }
725
726 if (opts.update_tracking and std.debug.runtime_safety) {
727 assert(func.register_manager.free_registers.eql(state.free_registers));
728 var used_reg_it = state.free_registers.iterator(.{ .kind = .unset });
729 while (used_reg_it.next()) |index|
730 assert(func.register_manager.registers[index] == state.registers[index]);
731 }
732}
733
734const Func = @This();
735
736const CallView = enum(u1) {
737 callee,
738 caller,
739};
740
741pub fn generate(
742 bin_file: *link.File,
743 pt: Zcu.PerThread,
744 src_loc: Zcu.LazySrcLoc,
745 func_index: InternPool.Index,
746 air: *const Air,
747 liveness: *const ?Air.Liveness,
748) CodeGenError!Mir {
749 const zcu = pt.zcu;
750 const gpa = zcu.gpa;
751 const ip = &zcu.intern_pool;
752 const func = zcu.funcInfo(func_index);
753 const fn_type = Type.fromInterned(func.ty);
754 const mod = zcu.navFileScope(func.owner_nav).mod.?;
755
756 var branch_stack = std.array_list.Managed(Branch).init(gpa);
757 defer {
758 assert(branch_stack.items.len == 1);
759 branch_stack.items[0].deinit(gpa);
760 branch_stack.deinit();
761 }
762 try branch_stack.append(.{});
763
764 var function: Func = .{
765 .gpa = gpa,
766 .air = air.*,
767 .pt = pt,
768 .mod = mod,
769 .bin_file = bin_file,
770 .liveness = liveness.*.?,
771 .target = &mod.resolved_target.result,
772 .owner = .{ .nav_index = func.owner_nav },
773 .args = undefined, // populated after `resolveCallingConventionValues`
774 .ret_mcv = undefined, // populated after `resolveCallingConventionValues`
775 .func_index = func_index,
776 .fn_type = fn_type,
777 .arg_index = 0,
778 .branch_stack = &branch_stack,
779 .src_loc = src_loc,
780 .end_di_line = func.rbrace_line,
781 .end_di_column = func.rbrace_column,
782 .scope_generation = 0,
783 .avl = null,
784 .vtype = null,
785 };
786 defer {
787 function.frame_allocs.deinit(gpa);
788 function.free_frame_indices.deinit(gpa);
789 function.frame_locs.deinit(gpa);
790 function.loops.deinit(gpa);
791 var block_it = function.blocks.valueIterator();
792 while (block_it.next()) |block| block.deinit(gpa);
793 function.blocks.deinit(gpa);
794 function.inst_tracking.deinit(gpa);
795 function.const_tracking.deinit(gpa);
796 function.exitlude_jump_relocs.deinit(gpa);
797 function.mir_instructions.deinit(gpa);
798 }
799
800 wip_mir_log.debug("{f}:", .{fmtNav(func.owner_nav, ip)});
801
802 try function.frame_allocs.resize(gpa, FrameIndex.named_count);
803 function.frame_allocs.set(
804 @intFromEnum(FrameIndex.stack_frame),
805 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
806 );
807 function.frame_allocs.set(
808 @intFromEnum(FrameIndex.call_frame),
809 FrameAlloc.init(.{ .size = 0, .alignment = .@"1" }),
810 );
811
812 const fn_info = zcu.typeToFunc(fn_type).?;
813 var call_info = function.resolveCallingConventionValues(fn_info, &.{}) catch |err| switch (err) {
814 error.CodegenFail => return error.CodegenFail,
815 else => |e| return e,
816 };
817
818 defer call_info.deinit(&function);
819
820 function.args = call_info.args;
821 function.ret_mcv = call_info.return_value;
822 function.frame_allocs.set(@intFromEnum(FrameIndex.ret_addr), FrameAlloc.init(.{
823 .size = Type.u64.abiSize(zcu),
824 .alignment = Type.u64.abiAlignment(zcu).min(call_info.stack_align),
825 }));
826 function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{
827 .size = Type.u64.abiSize(zcu),
828 .alignment = Alignment.min(
829 call_info.stack_align,
830 Alignment.fromNonzeroByteUnits(function.target.stackAlignment()),
831 ),
832 }));
833 function.frame_allocs.set(@intFromEnum(FrameIndex.args_frame), FrameAlloc.init(.{
834 .size = call_info.stack_byte_count,
835 .alignment = call_info.stack_align,
836 }));
837 function.frame_allocs.set(@intFromEnum(FrameIndex.spill_frame), FrameAlloc.init(.{
838 .size = 0,
839 .alignment = Type.u64.abiAlignment(zcu),
840 }));
841
842 function.gen() catch |err| switch (err) {
843 error.CodegenFail => return error.CodegenFail,
844 error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
845 else => |e| return e,
846 };
847
848 var mir: Mir = .{
849 .instructions = function.mir_instructions.toOwnedSlice(),
850 .frame_locs = function.frame_locs.toOwnedSlice(),
851 };
852 errdefer mir.deinit(gpa);
853 return mir;
854}
855
856pub fn generateLazy(
857 bin_file: *link.File,
858 pt: Zcu.PerThread,
859 src_loc: Zcu.LazySrcLoc,
860 lazy_sym: link.File.LazySymbol,
861 atom_index: u32,
862 w: *std.Io.Writer,
863 debug_output: link.File.DebugInfoOutput,
864) (CodeGenError || std.Io.Writer.Error)!void {
865 _ = atom_index;
866 const comp = bin_file.comp;
867 const gpa = comp.gpa;
868 const mod = comp.root_mod;
869
870 var function: Func = .{
871 .gpa = gpa,
872 .air = undefined,
873 .pt = pt,
874 .mod = mod,
875 .bin_file = bin_file,
876 .liveness = undefined,
877 .target = &mod.resolved_target.result,
878 .owner = .{ .lazy_sym = lazy_sym },
879 .args = undefined, // populated after `resolveCallingConventionValues`
880 .ret_mcv = undefined, // populated after `resolveCallingConventionValues`
881 .func_index = undefined,
882 .fn_type = undefined,
883 .arg_index = 0,
884 .branch_stack = undefined,
885 .src_loc = src_loc,
886 .end_di_line = undefined,
887 .end_di_column = undefined,
888 .scope_generation = 0,
889 .avl = null,
890 .vtype = null,
891 };
892 defer function.mir_instructions.deinit(gpa);
893
894 function.genLazy(lazy_sym) catch |err| switch (err) {
895 error.CodegenFail => return error.CodegenFail,
896 error.OutOfRegisters => return function.fail("ran out of registers (Zig compiler bug)", .{}),
897 else => |e| return e,
898 };
899
900 var mir: Mir = .{
901 .instructions = function.mir_instructions.toOwnedSlice(),
902 .frame_locs = function.frame_locs.toOwnedSlice(),
903 };
904 defer mir.deinit(gpa);
905
906 var emit: Emit = .{
907 .lower = .{
908 .pt = pt,
909 .allocator = gpa,
910 .mir = mir,
911 .cc = .auto,
912 .src_loc = src_loc,
913 .output_mode = comp.config.output_mode,
914 .link_mode = comp.config.link_mode,
915 .pic = mod.pic,
916 },
917 .bin_file = bin_file,
918 .debug_output = debug_output,
919 .w = w,
920 .prev_di_pc = undefined, // no debug info yet
921 .prev_di_line = undefined, // no debug info yet
922 .prev_di_column = undefined, // no debug info yet
923 };
924 defer emit.deinit();
925
926 emit.emitMir() catch |err| switch (err) {
927 error.LowerFail, error.EmitFail => return function.failMsg(emit.lower.err_msg.?),
928 error.InvalidInstruction => |e| return function.fail("emit MIR failed: {s} (Zig compiler bug)", .{@errorName(e)}),
929 else => |e| return e,
930 };
931}
932
933const FormatWipMirData = struct {
934 func: *Func,
935 inst: Mir.Inst.Index,
936};
937fn formatWipMir(data: FormatWipMirData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
938 const pt = data.func.pt;
939 const comp = pt.zcu.comp;
940 var lower: Lower = .{
941 .pt = pt,
942 .allocator = data.func.gpa,
943 .mir = .{
944 .instructions = data.func.mir_instructions.slice(),
945 .frame_locs = data.func.frame_locs.slice(),
946 },
947 .cc = .auto,
948 .src_loc = data.func.src_loc,
949 .output_mode = comp.config.output_mode,
950 .link_mode = comp.config.link_mode,
951 .pic = comp.root_mod.pic,
952 };
953 var first = true;
954 for ((lower.lowerMir(data.inst, .{ .allow_frame_locs = false }) catch |err| switch (err) {
955 error.LowerFail => {
956 defer {
957 lower.err_msg.?.deinit(data.func.gpa);
958 lower.err_msg = null;
959 }
960 try writer.writeAll(lower.err_msg.?.msg);
961 return;
962 },
963 error.OutOfMemory, error.InvalidInstruction => |e| {
964 try writer.writeAll(switch (e) {
965 error.OutOfMemory => "Out of memory",
966 error.InvalidInstruction => "CodeGen failed to find a viable instruction.",
967 });
968 return;
969 },
970 else => |e| return e,
971 }).insts) |lowered_inst| {
972 if (!first) try writer.writeAll("\ndebug(wip_mir): ");
973 try writer.print(" | {}", .{lowered_inst});
974 first = false;
975 }
976}
977fn fmtWipMir(func: *Func, inst: Mir.Inst.Index) std.fmt.Alt(FormatWipMirData, formatWipMir) {
978 return .{ .data = .{ .func = func, .inst = inst } };
979}
980
981const FormatNavData = struct {
982 ip: *const InternPool,
983 nav_index: InternPool.Nav.Index,
984};
985fn formatNav(data: FormatNavData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
986 try writer.print("{f}", .{data.ip.getNav(data.nav_index).fqn.fmt(data.ip)});
987}
988fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Alt(FormatNavData, formatNav) {
989 return .{ .data = .{
990 .ip = ip,
991 .nav_index = nav_index,
992 } };
993}
994
995const FormatAirData = struct {
996 func: *Func,
997 inst: Air.Inst.Index,
998};
999fn formatAir(data: FormatAirData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
1000 // Not acceptable implementation because it ignores `writer`:
1001 //data.func.air.dumpInst(data.inst, data.func.pt, data.func.liveness);
1002 _ = data;
1003 _ = writer;
1004 @panic("unimplemented");
1005}
1006fn fmtAir(func: *Func, inst: Air.Inst.Index) std.fmt.Alt(FormatAirData, formatAir) {
1007 return .{ .data = .{ .func = func, .inst = inst } };
1008}
1009
1010const FormatTrackingData = struct {
1011 func: *Func,
1012};
1013fn formatTracking(data: FormatTrackingData, writer: *std.Io.Writer) std.Io.Writer.Error!void {
1014 var it = data.func.inst_tracking.iterator();
1015 while (it.next()) |entry| try writer.print("\n%{d} = {f}", .{ entry.key_ptr.*, entry.value_ptr.* });
1016}
1017fn fmtTracking(func: *Func) std.fmt.Alt(FormatTrackingData, formatTracking) {
1018 return .{ .data = .{ .func = func } };
1019}
1020
1021fn addInst(func: *Func, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1022 const gpa = func.gpa;
1023 try func.mir_instructions.ensureUnusedCapacity(gpa, 1);
1024 const result_index: Mir.Inst.Index = @intCast(func.mir_instructions.len);
1025 func.mir_instructions.appendAssumeCapacity(inst);
1026 if (switch (inst.tag) {
1027 else => true,
1028 .pseudo_dbg_prologue_end,
1029 .pseudo_dbg_line_column,
1030 .pseudo_dbg_epilogue_begin,
1031 .pseudo_dead,
1032 => false,
1033 }) wip_mir_log.debug("{f}", .{func.fmtWipMir(result_index)});
1034 return result_index;
1035}
1036
1037fn addPseudo(func: *Func, mnem: Mnemonic) error{OutOfMemory}!Mir.Inst.Index {
1038 return func.addInst(.{
1039 .tag = mnem,
1040 .data = .none,
1041 });
1042}
1043
1044/// Returns a temporary register that contains the value of the `reg` csr.
1045///
1046/// Caller's duty to lock the return register is needed.
1047fn getCsr(func: *Func, csr: CSR) !Register {
1048 assert(func.hasFeature(.zicsr));
1049 const dst_reg = try func.register_manager.allocReg(null, func.regTempClassForType(Type.u64));
1050 _ = try func.addInst(.{
1051 .tag = .csrrs,
1052 .data = .{ .csr = .{
1053 .csr = csr,
1054 .rd = dst_reg,
1055 .rs1 = .x0,
1056 } },
1057 });
1058 return dst_reg;
1059}
1060
1061fn setVl(func: *Func, dst_reg: Register, avl: u64, options: bits.VType) !void {
1062 if (func.avl == avl) if (func.vtype) |vtype| {
1063 // it's already set, we don't need to do anything
1064 if (@as(u8, @bitCast(vtype)) == @as(u8, @bitCast(options))) return;
1065 };
1066
1067 func.avl = avl;
1068 func.vtype = options;
1069
1070 if (avl == 0) {
1071 // the caller means to do "vsetvli zero, zero ..." which keeps the avl to whatever it was before
1072 const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options));
1073 _ = try func.addInst(.{
1074 .tag = .vsetvli,
1075 .data = .{ .i_type = .{
1076 .rd = dst_reg,
1077 .rs1 = .zero,
1078 .imm12 = Immediate.u(options_int),
1079 } },
1080 });
1081 } else {
1082 // if the avl can fit into u5 we can use vsetivli otherwise use vsetvli
1083 if (avl <= std.math.maxInt(u5)) {
1084 const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options));
1085 _ = try func.addInst(.{
1086 .tag = .vsetivli,
1087 .data = .{
1088 .i_type = .{
1089 .rd = dst_reg,
1090 .rs1 = @enumFromInt(avl),
1091 .imm12 = Immediate.u(options_int),
1092 },
1093 },
1094 });
1095 } else {
1096 const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options));
1097 const temp_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = avl });
1098 _ = try func.addInst(.{
1099 .tag = .vsetvli,
1100 .data = .{ .i_type = .{
1101 .rd = dst_reg,
1102 .rs1 = temp_reg,
1103 .imm12 = Immediate.u(options_int),
1104 } },
1105 });
1106 }
1107 }
1108}
1109
1110const required_features = [_]Target.riscv.Feature{
1111 .d,
1112 .m,
1113 .a,
1114 .zicsr,
1115 .v,
1116 .zbb,
1117};
1118
1119fn gen(func: *Func) !void {
1120 const pt = func.pt;
1121 const zcu = pt.zcu;
1122 const fn_info = zcu.typeToFunc(func.fn_type).?;
1123
1124 inline for (required_features) |feature| {
1125 if (!func.hasFeature(feature)) {
1126 return func.fail(
1127 "target missing required feature {s}",
1128 .{@tagName(feature)},
1129 );
1130 }
1131 }
1132
1133 if (fn_info.cc != .naked) {
1134 _ = try func.addPseudo(.pseudo_dbg_prologue_end);
1135
1136 const backpatch_stack_alloc = try func.addPseudo(.pseudo_dead);
1137 const backpatch_ra_spill = try func.addPseudo(.pseudo_dead);
1138 const backpatch_fp_spill = try func.addPseudo(.pseudo_dead);
1139 const backpatch_fp_add = try func.addPseudo(.pseudo_dead);
1140 const backpatch_spill_callee_preserved_regs = try func.addPseudo(.pseudo_dead);
1141
1142 switch (func.ret_mcv.long) {
1143 .none, .unreach => {},
1144 .indirect => {
1145 // The address where to store the return value for the caller is in a
1146 // register which the callee is free to clobber. Therefore, we purposely
1147 // spill it to stack immediately.
1148 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(Type.u64, zcu));
1149 try func.genSetMem(
1150 .{ .frame = frame_index },
1151 0,
1152 Type.u64,
1153 func.ret_mcv.long.address().offset(-func.ret_mcv.short.indirect.off),
1154 );
1155 func.ret_mcv.long = .{ .load_frame = .{ .index = frame_index } };
1156 tracking_log.debug("spill {} to {}", .{ func.ret_mcv.long, frame_index });
1157 },
1158 else => unreachable,
1159 }
1160
1161 try func.genBody(func.air.getMainBody());
1162
1163 for (func.exitlude_jump_relocs.items) |jmp_reloc| {
1164 func.mir_instructions.items(.data)[jmp_reloc].j_type.inst =
1165 @intCast(func.mir_instructions.len);
1166 }
1167
1168 _ = try func.addPseudo(.pseudo_dbg_epilogue_begin);
1169
1170 const backpatch_restore_callee_preserved_regs = try func.addPseudo(.pseudo_dead);
1171 const backpatch_ra_restore = try func.addPseudo(.pseudo_dead);
1172 const backpatch_fp_restore = try func.addPseudo(.pseudo_dead);
1173 const backpatch_stack_alloc_restore = try func.addPseudo(.pseudo_dead);
1174
1175 // ret
1176 _ = try func.addInst(.{
1177 .tag = .jalr,
1178 .data = .{ .i_type = .{
1179 .rd = .zero,
1180 .rs1 = .ra,
1181 .imm12 = Immediate.s(0),
1182 } },
1183 });
1184
1185 const frame_layout = try func.computeFrameLayout();
1186 const need_save_reg = frame_layout.save_reg_list.count() > 0;
1187
1188 func.mir_instructions.set(backpatch_stack_alloc, .{
1189 .tag = .addi,
1190 .data = .{ .i_type = .{
1191 .rd = .sp,
1192 .rs1 = .sp,
1193 .imm12 = Immediate.s(-@as(i32, @intCast(frame_layout.stack_adjust))),
1194 } },
1195 });
1196 func.mir_instructions.set(backpatch_ra_spill, .{
1197 .tag = .pseudo_store_rm,
1198 .data = .{ .rm = .{
1199 .r = .ra,
1200 .m = .{
1201 .base = .{ .frame = .ret_addr },
1202 .mod = .{ .size = .dword, .unsigned = false },
1203 },
1204 } },
1205 });
1206 func.mir_instructions.set(backpatch_ra_restore, .{
1207 .tag = .pseudo_load_rm,
1208 .data = .{ .rm = .{
1209 .r = .ra,
1210 .m = .{
1211 .base = .{ .frame = .ret_addr },
1212 .mod = .{ .size = .dword, .unsigned = false },
1213 },
1214 } },
1215 });
1216 func.mir_instructions.set(backpatch_fp_spill, .{
1217 .tag = .pseudo_store_rm,
1218 .data = .{ .rm = .{
1219 .r = .s0,
1220 .m = .{
1221 .base = .{ .frame = .base_ptr },
1222 .mod = .{ .size = .dword, .unsigned = false },
1223 },
1224 } },
1225 });
1226 func.mir_instructions.set(backpatch_fp_restore, .{
1227 .tag = .pseudo_load_rm,
1228 .data = .{ .rm = .{
1229 .r = .s0,
1230 .m = .{
1231 .base = .{ .frame = .base_ptr },
1232 .mod = .{ .size = .dword, .unsigned = false },
1233 },
1234 } },
1235 });
1236 func.mir_instructions.set(backpatch_fp_add, .{
1237 .tag = .addi,
1238 .data = .{ .i_type = .{
1239 .rd = .s0,
1240 .rs1 = .sp,
1241 .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
1242 } },
1243 });
1244 func.mir_instructions.set(backpatch_stack_alloc_restore, .{
1245 .tag = .addi,
1246 .data = .{ .i_type = .{
1247 .rd = .sp,
1248 .rs1 = .sp,
1249 .imm12 = Immediate.s(@intCast(frame_layout.stack_adjust)),
1250 } },
1251 });
1252
1253 if (need_save_reg) {
1254 func.mir_instructions.set(backpatch_spill_callee_preserved_regs, .{
1255 .tag = .pseudo_spill_regs,
1256 .data = .{ .reg_list = frame_layout.save_reg_list },
1257 });
1258
1259 func.mir_instructions.set(backpatch_restore_callee_preserved_regs, .{
1260 .tag = .pseudo_restore_regs,
1261 .data = .{ .reg_list = frame_layout.save_reg_list },
1262 });
1263 }
1264 } else {
1265 _ = try func.addPseudo(.pseudo_dbg_prologue_end);
1266 try func.genBody(func.air.getMainBody());
1267 _ = try func.addPseudo(.pseudo_dbg_epilogue_begin);
1268 }
1269
1270 // Drop them off at the rbrace.
1271 _ = try func.addInst(.{
1272 .tag = .pseudo_dbg_line_column,
1273 .data = .{ .pseudo_dbg_line_column = .{
1274 .line = func.end_di_line,
1275 .column = func.end_di_column,
1276 } },
1277 });
1278}
1279
1280fn genLazy(func: *Func, lazy_sym: link.File.LazySymbol) InnerError!void {
1281 const pt = func.pt;
1282 const zcu = pt.zcu;
1283 const ip = &zcu.intern_pool;
1284 switch (Type.fromInterned(lazy_sym.ty).zigTypeTag(zcu)) {
1285 .@"enum" => {
1286 const enum_ty = Type.fromInterned(lazy_sym.ty);
1287 wip_mir_log.debug("{f}.@tagName:", .{enum_ty.fmt(pt)});
1288
1289 const param_regs = abi.Registers.Integer.function_arg_regs;
1290 const ret_reg = param_regs[0];
1291 const enum_mcv: MCValue = .{ .register = param_regs[1] };
1292
1293 const exitlude_jump_relocs = try func.gpa.alloc(Mir.Inst.Index, enum_ty.enumFieldCount(zcu));
1294 defer func.gpa.free(exitlude_jump_relocs);
1295
1296 const data_reg, const data_lock = try func.allocReg(.int);
1297 defer func.register_manager.unlockReg(data_lock);
1298
1299 const elf_file = func.bin_file.cast(.elf).?;
1300 const zo = elf_file.zigObjectPtr().?;
1301 const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, .{
1302 .kind = .const_data,
1303 .ty = enum_ty.toIntern(),
1304 }) catch |err|
1305 return func.fail("{s} creating lazy symbol", .{@errorName(err)});
1306
1307 try func.genSetReg(Type.u64, data_reg, .{ .lea_symbol = .{ .sym = sym_index } });
1308
1309 const cmp_reg, const cmp_lock = try func.allocReg(.int);
1310 defer func.register_manager.unlockReg(cmp_lock);
1311
1312 var data_off: i32 = 0;
1313 const tag_names = enum_ty.enumFields(zcu);
1314 for (exitlude_jump_relocs, 0..) |*exitlude_jump_reloc, tag_index| {
1315 const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
1316 const tag_val = try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index));
1317 const tag_mcv = try func.genTypedValue(tag_val);
1318
1319 _ = try func.genBinOp(
1320 .cmp_neq,
1321 enum_mcv,
1322 enum_ty,
1323 tag_mcv,
1324 enum_ty,
1325 cmp_reg,
1326 );
1327 const skip_reloc = try func.condBr(Type.bool, .{ .register = cmp_reg });
1328
1329 try func.genSetMem(
1330 .{ .reg = ret_reg },
1331 0,
1332 Type.u64,
1333 .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
1334 );
1335
1336 try func.genSetMem(
1337 .{ .reg = ret_reg },
1338 8,
1339 Type.u64,
1340 .{ .immediate = tag_name_len },
1341 );
1342
1343 exitlude_jump_reloc.* = try func.addInst(.{
1344 .tag = .pseudo_j,
1345 .data = .{ .j_type = .{
1346 .rd = .zero,
1347 .inst = undefined,
1348 } },
1349 });
1350 func.performReloc(skip_reloc);
1351
1352 data_off += @intCast(tag_name_len + 1);
1353 }
1354
1355 try func.airTrap();
1356
1357 for (exitlude_jump_relocs) |reloc| func.performReloc(reloc);
1358
1359 _ = try func.addInst(.{
1360 .tag = .jalr,
1361 .data = .{ .i_type = .{
1362 .rd = .zero,
1363 .rs1 = .ra,
1364 .imm12 = Immediate.s(0),
1365 } },
1366 });
1367 },
1368 else => return func.fail(
1369 "TODO implement {s} for {f}",
1370 .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
1371 ),
1372 }
1373}
1374
1375fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1376 const pt = func.pt;
1377 const zcu = pt.zcu;
1378 const ip = &zcu.intern_pool;
1379 const air_tags = func.air.instructions.items(.tag);
1380
1381 for (body) |inst| {
1382 if (func.liveness.isUnused(inst) and !func.air.mustLower(inst, ip)) continue;
1383 wip_mir_log.debug("{f}", .{func.fmtAir(inst)});
1384 verbose_tracking_log.debug("{f}", .{func.fmtTracking()});
1385
1386 const old_air_bookkeeping = func.air_bookkeeping;
1387 try func.ensureProcessDeathCapacity(Air.Liveness.bpi);
1388
1389 func.reused_operands = @TypeOf(func.reused_operands).initEmpty();
1390 try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1);
1391 const tag = air_tags[@intFromEnum(inst)];
1392 switch (tag) {
1393 // zig fmt: off
1394
1395 // No "scalarize" legalizations are enabled, so these instructions never appear.
1396 .legalize_vec_elem_val => unreachable,
1397 .legalize_vec_store_elem => unreachable,
1398 // No soft float legalizations are enabled.
1399 .legalize_compiler_rt_call => unreachable,
1400
1401 .add,
1402 .add_wrap,
1403 .sub,
1404 .sub_wrap,
1405
1406 .add_sat,
1407
1408 .mul,
1409 .mul_wrap,
1410 .div_trunc,
1411 .div_exact,
1412 .rem,
1413
1414 .shl, .shl_exact,
1415 .shr, .shr_exact,
1416
1417 .bool_and,
1418 .bool_or,
1419 .bit_and,
1420 .bit_or,
1421
1422 .xor,
1423
1424 .min,
1425 .max,
1426 => try func.airBinOp(inst, tag),
1427
1428
1429 .ptr_add,
1430 .ptr_sub => try func.airPtrArithmetic(inst, tag),
1431
1432 .mod,
1433 .div_float,
1434 .div_floor,
1435 => return func.fail("TODO: {s}", .{@tagName(tag)}),
1436
1437 .sqrt,
1438 .sin,
1439 .cos,
1440 .tan,
1441 .exp,
1442 .exp2,
1443 .log,
1444 .log2,
1445 .log10,
1446 .floor,
1447 .ceil,
1448 .round,
1449 .trunc_float,
1450 .neg,
1451 => try func.airUnaryMath(inst, tag),
1452
1453 .add_with_overflow => try func.airAddWithOverflow(inst),
1454 .sub_with_overflow => try func.airSubWithOverflow(inst),
1455 .mul_with_overflow => try func.airMulWithOverflow(inst),
1456 .shl_with_overflow => try func.airShlWithOverflow(inst),
1457
1458
1459 .sub_sat => try func.airSubSat(inst),
1460 .mul_sat => try func.airMulSat(inst),
1461 .shl_sat => try func.airShlSat(inst),
1462
1463 .add_safe,
1464 .sub_safe,
1465 .mul_safe,
1466 .intcast_safe,
1467 .int_from_float_safe,
1468 .int_from_float_optimized_safe,
1469 => return func.fail("TODO implement safety_checked_instructions", .{}),
1470
1471 .cmp_lt,
1472 .cmp_lte,
1473 .cmp_eq,
1474 .cmp_gte,
1475 .cmp_gt,
1476 .cmp_neq,
1477 => try func.airCmp(inst, tag),
1478
1479 .cmp_vector => try func.airCmpVector(inst),
1480 .cmp_lt_errors_len => try func.airCmpLtErrorsLen(inst),
1481
1482 .slice => try func.airSlice(inst),
1483 .array_to_slice => try func.airArrayToSlice(inst),
1484
1485 .slice_ptr => try func.airSlicePtr(inst),
1486 .slice_len => try func.airSliceLen(inst),
1487
1488 .alloc => try func.airAlloc(inst),
1489 .ret_ptr => try func.airRetPtr(inst),
1490 .arg => try func.airArg(inst),
1491 .assembly => try func.airAsm(inst),
1492 .bitcast => try func.airBitCast(inst),
1493 .block => try func.airBlock(inst),
1494 .br => try func.airBr(inst),
1495 .repeat => try func.airRepeat(inst),
1496 .switch_dispatch => try func.airSwitchDispatch(inst),
1497 .trap => try func.airTrap(),
1498 .breakpoint => try func.airBreakpoint(),
1499 .ret_addr => try func.airRetAddr(inst),
1500 .frame_addr => try func.airFrameAddress(inst),
1501 .cond_br => try func.airCondBr(inst),
1502 .dbg_stmt => try func.airDbgStmt(inst),
1503 .dbg_empty_stmt => func.finishAirBookkeeping(),
1504 .fptrunc => try func.airFptrunc(inst),
1505 .fpext => try func.airFpext(inst),
1506 .intcast => try func.airIntCast(inst),
1507 .trunc => try func.airTrunc(inst),
1508 .is_non_null => try func.airIsNonNull(inst),
1509 .is_non_null_ptr => try func.airIsNonNullPtr(inst),
1510 .is_null => try func.airIsNull(inst),
1511 .is_null_ptr => try func.airIsNullPtr(inst),
1512 .is_non_err => try func.airIsNonErr(inst),
1513 .is_non_err_ptr => try func.airIsNonErrPtr(inst),
1514 .is_err => try func.airIsErr(inst),
1515 .is_err_ptr => try func.airIsErrPtr(inst),
1516 .load => try func.airLoad(inst),
1517 .loop => try func.airLoop(inst),
1518 .not => try func.airNot(inst),
1519 .ret => try func.airRet(inst, false),
1520 .ret_safe => try func.airRet(inst, true),
1521 .ret_load => try func.airRetLoad(inst),
1522 .store => try func.airStore(inst, false),
1523 .store_safe => try func.airStore(inst, true),
1524 .struct_field_ptr=> try func.airStructFieldPtr(inst),
1525 .struct_field_val=> try func.airStructFieldVal(inst),
1526 .float_from_int => try func.airFloatFromInt(inst),
1527 .int_from_float => try func.airIntFromFloat(inst),
1528 .cmpxchg_strong => try func.airCmpxchg(inst, .strong),
1529 .cmpxchg_weak => try func.airCmpxchg(inst, .weak),
1530 .atomic_rmw => try func.airAtomicRmw(inst),
1531 .atomic_load => try func.airAtomicLoad(inst),
1532 .memcpy => try func.airMemcpy(inst),
1533 .memmove => try func.airMemmove(inst),
1534 .memset => try func.airMemset(inst, false),
1535 .memset_safe => try func.airMemset(inst, true),
1536 .set_union_tag => try func.airSetUnionTag(inst),
1537 .get_union_tag => try func.airGetUnionTag(inst),
1538 .clz => try func.airClz(inst),
1539 .ctz => try func.airCtz(inst),
1540 .popcount => try func.airPopcount(inst),
1541 .abs => try func.airAbs(inst),
1542 .byte_swap => try func.airByteSwap(inst),
1543 .bit_reverse => try func.airBitReverse(inst),
1544 .tag_name => try func.airTagName(inst),
1545 .error_name => try func.airErrorName(inst),
1546 .splat => try func.airSplat(inst),
1547 .select => try func.airSelect(inst),
1548 .shuffle_one => try func.airShuffleOne(inst),
1549 .shuffle_two => try func.airShuffleTwo(inst),
1550 .reduce => try func.airReduce(inst),
1551 .aggregate_init => try func.airAggregateInit(inst),
1552 .union_init => try func.airUnionInit(inst),
1553 .prefetch => try func.airPrefetch(inst),
1554 .mul_add => try func.airMulAdd(inst),
1555 .addrspace_cast => return func.fail("TODO: addrspace_cast", .{}),
1556
1557 .@"try" => try func.airTry(inst),
1558 .try_cold => try func.airTry(inst),
1559 .try_ptr => return func.fail("TODO: try_ptr", .{}),
1560 .try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}),
1561
1562 .dbg_var_ptr,
1563 .dbg_var_val,
1564 .dbg_arg_inline,
1565 => try func.airDbgVar(inst),
1566
1567 .dbg_inline_block => try func.airDbgInlineBlock(inst),
1568
1569 .call => try func.airCall(inst, .auto),
1570 .call_always_tail => try func.airCall(inst, .always_tail),
1571 .call_never_tail => try func.airCall(inst, .never_tail),
1572 .call_never_inline => try func.airCall(inst, .never_inline),
1573
1574 .atomic_store_unordered => try func.airAtomicStore(inst, .unordered),
1575 .atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic),
1576 .atomic_store_release => try func.airAtomicStore(inst, .release),
1577 .atomic_store_seq_cst => try func.airAtomicStore(inst, .seq_cst),
1578 .struct_field_ptr_index_0 => try func.airStructFieldPtrIndex(inst, 0),
1579 .struct_field_ptr_index_1 => try func.airStructFieldPtrIndex(inst, 1),
1580 .struct_field_ptr_index_2 => try func.airStructFieldPtrIndex(inst, 2),
1581 .struct_field_ptr_index_3 => try func.airStructFieldPtrIndex(inst, 3),
1582
1583 .field_parent_ptr => try func.airFieldParentPtr(inst),
1584
1585 .switch_br => try func.airSwitchBr(inst),
1586 .loop_switch_br => try func.airLoopSwitchBr(inst),
1587
1588 .ptr_slice_len_ptr => try func.airPtrSliceLenPtr(inst),
1589 .ptr_slice_ptr_ptr => try func.airPtrSlicePtrPtr(inst),
1590
1591 .array_elem_val => try func.airArrayElemVal(inst),
1592
1593 .slice_elem_val => try func.airSliceElemVal(inst),
1594 .slice_elem_ptr => try func.airSliceElemPtr(inst),
1595
1596 .ptr_elem_val => try func.airPtrElemVal(inst),
1597 .ptr_elem_ptr => try func.airPtrElemPtr(inst),
1598
1599 .inferred_alloc, .inferred_alloc_comptime => unreachable,
1600 .unreach => func.finishAirBookkeeping(),
1601
1602 .optional_payload => try func.airOptionalPayload(inst),
1603 .optional_payload_ptr => try func.airOptionalPayloadPtr(inst),
1604 .optional_payload_ptr_set => try func.airOptionalPayloadPtrSet(inst),
1605 .unwrap_errunion_err => try func.airUnwrapErrErr(inst),
1606 .unwrap_errunion_payload => try func.airUnwrapErrPayload(inst),
1607 .unwrap_errunion_err_ptr => try func.airUnwrapErrErrPtr(inst),
1608 .unwrap_errunion_payload_ptr=> try func.airUnwrapErrPayloadPtr(inst),
1609 .errunion_payload_ptr_set => try func.airErrUnionPayloadPtrSet(inst),
1610 .err_return_trace => try func.airErrReturnTrace(inst),
1611 .set_err_return_trace => try func.airSetErrReturnTrace(inst),
1612 .save_err_return_trace_index=> try func.airSaveErrReturnTraceIndex(inst),
1613
1614 .wrap_optional => try func.airWrapOptional(inst),
1615 .wrap_errunion_payload => try func.airWrapErrUnionPayload(inst),
1616 .wrap_errunion_err => try func.airWrapErrUnionErr(inst),
1617
1618 .runtime_nav_ptr => try func.airRuntimeNavPtr(inst),
1619
1620 .add_optimized,
1621 .sub_optimized,
1622 .mul_optimized,
1623 .div_float_optimized,
1624 .div_trunc_optimized,
1625 .div_floor_optimized,
1626 .div_exact_optimized,
1627 .rem_optimized,
1628 .mod_optimized,
1629 .neg_optimized,
1630 .cmp_lt_optimized,
1631 .cmp_lte_optimized,
1632 .cmp_eq_optimized,
1633 .cmp_gte_optimized,
1634 .cmp_gt_optimized,
1635 .cmp_neq_optimized,
1636 .cmp_vector_optimized,
1637 .reduce_optimized,
1638 .int_from_float_optimized,
1639 => return func.fail("TODO implement optimized float mode", .{}),
1640
1641 .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}),
1642 .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}),
1643
1644 .c_va_arg => return func.fail("TODO implement c_va_arg", .{}),
1645 .c_va_copy => return func.fail("TODO implement c_va_copy", .{}),
1646 .c_va_end => return func.fail("TODO implement c_va_end", .{}),
1647 .c_va_start => return func.fail("TODO implement c_va_start", .{}),
1648
1649 .wasm_memory_size => unreachable,
1650 .wasm_memory_grow => unreachable,
1651
1652 .work_item_id => unreachable,
1653 .work_group_size => unreachable,
1654 .work_group_id => unreachable,
1655 // zig fmt: on
1656 }
1657
1658 assert(!func.register_manager.lockedRegsExist());
1659
1660 if (std.debug.runtime_safety) {
1661 if (func.air_bookkeeping < old_air_bookkeeping + 1) {
1662 std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[@intFromEnum(inst)] });
1663 }
1664
1665 { // check consistency of tracked registers
1666 var it = func.register_manager.free_registers.iterator(.{ .kind = .unset });
1667 while (it.next()) |index| {
1668 const tracked_inst = func.register_manager.registers[index];
1669 tracking_log.debug("tracked inst: {f}", .{tracked_inst});
1670 const tracking = func.getResolvedInstValue(tracked_inst);
1671 for (tracking.getRegs()) |reg| {
1672 if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break;
1673 } else return std.debug.panic(
1674 \\%{f} takes up these regs: {any}, however this regs {any}, don't use it
1675 , .{ tracked_inst, tracking.getRegs(), RegisterManager.regAtTrackedIndex(@intCast(index)) });
1676 }
1677 }
1678 }
1679 }
1680 verbose_tracking_log.debug("{f}", .{func.fmtTracking()});
1681}
1682
1683fn getValue(func: *Func, value: MCValue, inst: ?Air.Inst.Index) !void {
1684 for (value.getRegs()) |reg| try func.register_manager.getReg(reg, inst);
1685}
1686
1687fn getValueIfFree(func: *Func, value: MCValue, inst: ?Air.Inst.Index) void {
1688 for (value.getRegs()) |reg| if (func.register_manager.isRegFree(reg))
1689 func.register_manager.getRegAssumeFree(reg, inst);
1690}
1691
1692fn freeValue(func: *Func, value: MCValue) !void {
1693 switch (value) {
1694 .register => |reg| func.register_manager.freeReg(reg),
1695 .register_pair => |regs| for (regs) |reg| func.register_manager.freeReg(reg),
1696 .register_offset => |reg_off| func.register_manager.freeReg(reg_off.reg),
1697 else => {}, // TODO process stack allocation death
1698 }
1699}
1700
1701fn feed(func: *Func, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) !void {
1702 if (bt.feed()) if (operand.toIndex()) |inst| {
1703 log.debug("feed inst: %{f}", .{inst});
1704 try func.processDeath(inst);
1705 };
1706}
1707
1708/// Asserts there is already capacity to insert into top branch inst_table.
1709fn processDeath(func: *Func, inst: Air.Inst.Index) !void {
1710 try func.inst_tracking.getPtr(inst).?.die(func, inst);
1711}
1712
1713/// Called when there are no operands, and the instruction is always unreferenced.
1714fn finishAirBookkeeping(func: *Func) void {
1715 if (std.debug.runtime_safety) {
1716 func.air_bookkeeping += 1;
1717 }
1718}
1719
1720fn finishAirResult(func: *Func, inst: Air.Inst.Index, result: MCValue) void {
1721 if (func.liveness.isUnused(inst)) switch (result) {
1722 .none, .dead, .unreach => {},
1723 // Why didn't the result die?
1724 .register => |r| if (r != .zero) unreachable,
1725 else => unreachable,
1726 } else {
1727 switch (result) {
1728 .register => |r| if (r == .zero) unreachable, // Why did we discard a used result?
1729 else => {},
1730 }
1731
1732 tracking_log.debug("%{d} => {} (birth)", .{ inst, result });
1733 func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result));
1734 // In some cases, an operand may be reused as the result.
1735 // If that operand died and was a register, it was freed by
1736 // processDeath, so we have to "re-allocate" the register.
1737 func.getValueIfFree(result, inst);
1738 }
1739 func.finishAirBookkeeping();
1740}
1741
1742fn finishAir(
1743 func: *Func,
1744 inst: Air.Inst.Index,
1745 result: MCValue,
1746 operands: [Air.Liveness.bpi - 1]Air.Inst.Ref,
1747) !void {
1748 const tomb_bits = func.liveness.getTombBits(inst);
1749 for (0.., operands) |op_index, op| {
1750 if (tomb_bits & @as(Air.Liveness.Bpi, 1) << @intCast(op_index) == 0) continue;
1751 if (func.reused_operands.isSet(op_index)) continue;
1752 try func.processDeath(op.toIndexAllowNone() orelse continue);
1753 }
1754 func.finishAirResult(inst, result);
1755}
1756
1757const FrameLayout = struct {
1758 stack_adjust: i12,
1759 save_reg_list: Mir.RegisterList,
1760};
1761
1762fn setFrameLoc(
1763 func: *Func,
1764 frame_index: FrameIndex,
1765 base: Register,
1766 offset: *i32,
1767 comptime aligned: bool,
1768) void {
1769 const frame_i = @intFromEnum(frame_index);
1770 if (aligned) {
1771 const alignment: InternPool.Alignment = func.frame_allocs.items(.abi_align)[frame_i];
1772 offset.* = math.sign(offset.*) * @as(i32, @intCast(alignment.backward(@intCast(@abs(offset.*)))));
1773 }
1774 func.frame_locs.set(frame_i, .{ .base = base, .disp = offset.* });
1775 offset.* += func.frame_allocs.items(.abi_size)[frame_i];
1776}
1777
1778fn computeFrameLayout(func: *Func) !FrameLayout {
1779 const frame_allocs_len = func.frame_allocs.len;
1780 try func.frame_locs.resize(func.gpa, frame_allocs_len);
1781 const stack_frame_order = try func.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
1782 defer func.gpa.free(stack_frame_order);
1783
1784 const frame_size = func.frame_allocs.items(.abi_size);
1785 const frame_align = func.frame_allocs.items(.abi_align);
1786
1787 for (stack_frame_order, FrameIndex.named_count..) |*frame_order, frame_index|
1788 frame_order.* = @enumFromInt(frame_index);
1789
1790 {
1791 const SortContext = struct {
1792 frame_align: @TypeOf(frame_align),
1793 pub fn lessThan(context: @This(), lhs: FrameIndex, rhs: FrameIndex) bool {
1794 return context.frame_align[@intFromEnum(lhs)].compare(.gt, context.frame_align[@intFromEnum(rhs)]);
1795 }
1796 };
1797 const sort_context = SortContext{ .frame_align = frame_align };
1798 mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan);
1799 }
1800
1801 var save_reg_list = Mir.RegisterList{};
1802 for (abi.Registers.all_preserved) |reg| {
1803 if (func.register_manager.isRegAllocated(reg)) {
1804 save_reg_list.push(&abi.Registers.all_preserved, reg);
1805 }
1806 }
1807
1808 const total_alloc_size: i32 = blk: {
1809 var i: i32 = 0;
1810 for (stack_frame_order) |frame_index| {
1811 i += frame_size[@intFromEnum(frame_index)];
1812 }
1813 break :blk i;
1814 };
1815
1816 const saved_reg_size = save_reg_list.size();
1817 frame_size[@intFromEnum(FrameIndex.spill_frame)] = @intCast(saved_reg_size);
1818
1819 // The total frame size is calculated by the amount of s registers you need to save * 8, as each
1820 // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0
1821 // register. Finally we align the frame size to the alignment of the base pointer.
1822 const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)];
1823 const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)];
1824 const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)];
1825
1826 // TODO: this 64 should be a 16, but we were clobbering the top and bottom of the frame.
1827 // maybe everything can go from the bottom?
1828 const acc_frame_size: i32 = std.mem.alignForward(
1829 i32,
1830 total_alloc_size + 64 + args_frame_size + spill_frame_size + call_frame_size,
1831 @intCast(frame_align[@intFromEnum(FrameIndex.base_ptr)].toByteUnits().?),
1832 );
1833 log.debug("frame size: {d}", .{acc_frame_size});
1834
1835 // store the ra at total_size - 8, so it's the very first thing in the stack
1836 // relative to the fp
1837 func.frame_locs.set(
1838 @intFromEnum(FrameIndex.ret_addr),
1839 .{ .base = .sp, .disp = acc_frame_size - 8 },
1840 );
1841 func.frame_locs.set(
1842 @intFromEnum(FrameIndex.base_ptr),
1843 .{ .base = .sp, .disp = acc_frame_size - 16 },
1844 );
1845
1846 // now we grow the stack frame from the bottom of total frame in order to
1847 // not need to know the size of the first allocation. Stack offsets point at the "bottom"
1848 // of variables.
1849 var s0_offset: i32 = -acc_frame_size;
1850 func.setFrameLoc(.stack_frame, .s0, &s0_offset, true);
1851 for (stack_frame_order) |frame_index| func.setFrameLoc(frame_index, .s0, &s0_offset, true);
1852 func.setFrameLoc(.args_frame, .s0, &s0_offset, true);
1853 func.setFrameLoc(.call_frame, .s0, &s0_offset, true);
1854 func.setFrameLoc(.spill_frame, .s0, &s0_offset, true);
1855
1856 return .{
1857 .stack_adjust = @intCast(acc_frame_size),
1858 .save_reg_list = save_reg_list,
1859 };
1860}
1861
1862fn ensureProcessDeathCapacity(func: *Func, additional_count: usize) !void {
1863 const table = &func.branch_stack.items[func.branch_stack.items.len - 1].inst_table;
1864 try table.ensureUnusedCapacity(func.gpa, additional_count);
1865}
1866
1867fn memSize(func: *Func, ty: Type) Memory.Size {
1868 const pt = func.pt;
1869 const zcu = pt.zcu;
1870 return switch (ty.zigTypeTag(zcu)) {
1871 .float => Memory.Size.fromBitSize(ty.floatBits(func.target)),
1872 else => Memory.Size.fromByteSize(ty.abiSize(zcu)),
1873 };
1874}
1875
1876fn splitType(func: *Func, ty: Type) ![2]Type {
1877 const zcu = func.pt.zcu;
1878 const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
1879 var parts: [2]Type = undefined;
1880 if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| {
1881 part.* = switch (class) {
1882 .integer => switch (part_i) {
1883 0 => Type.u64,
1884 1 => part: {
1885 const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?;
1886 const elem_ty = try func.pt.intType(.unsigned, @intCast(elem_size * 8));
1887 break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) {
1888 1 => elem_ty,
1889 else => |len| try func.pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
1890 };
1891 },
1892 else => unreachable,
1893 },
1894 else => return func.fail("TODO: splitType class {}", .{class}),
1895 };
1896 } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts;
1897 return func.fail("TODO implement splitType for {f}", .{ty.fmt(func.pt)});
1898}
1899
1900/// Truncates the value in the register in place.
1901/// Clobbers any remaining bits.
1902fn truncateRegister(func: *Func, ty: Type, reg: Register) !void {
1903 const pt = func.pt;
1904 const zcu = pt.zcu;
1905 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
1906 .signedness = .unsigned,
1907 .bits = @intCast(ty.bitSize(zcu)),
1908 };
1909 assert(reg.class() == .int);
1910
1911 const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return;
1912 switch (int_info.signedness) {
1913 .signed => {
1914 _ = try func.addInst(.{
1915 .tag = .slli,
1916
1917 .data = .{
1918 .i_type = .{
1919 .rd = reg,
1920 .rs1 = reg,
1921 .imm12 = Immediate.u(shift),
1922 },
1923 },
1924 });
1925 _ = try func.addInst(.{
1926 .tag = .srai,
1927
1928 .data = .{
1929 .i_type = .{
1930 .rd = reg,
1931 .rs1 = reg,
1932 .imm12 = Immediate.u(shift),
1933 },
1934 },
1935 });
1936 },
1937 .unsigned => {
1938 const mask = ~@as(u64, 0) >> shift;
1939 if (mask < 256) {
1940 _ = try func.addInst(.{
1941 .tag = .andi,
1942
1943 .data = .{
1944 .i_type = .{
1945 .rd = reg,
1946 .rs1 = reg,
1947 .imm12 = Immediate.u(@intCast(mask)),
1948 },
1949 },
1950 });
1951 } else {
1952 _ = try func.addInst(.{
1953 .tag = .slli,
1954
1955 .data = .{
1956 .i_type = .{
1957 .rd = reg,
1958 .rs1 = reg,
1959 .imm12 = Immediate.u(shift),
1960 },
1961 },
1962 });
1963 _ = try func.addInst(.{
1964 .tag = .srli,
1965
1966 .data = .{
1967 .i_type = .{
1968 .rd = reg,
1969 .rs1 = reg,
1970 .imm12 = Immediate.u(shift),
1971 },
1972 },
1973 });
1974 }
1975 },
1976 }
1977}
1978
1979fn allocFrameIndex(func: *Func, alloc: FrameAlloc) !FrameIndex {
1980 const frame_allocs_slice = func.frame_allocs.slice();
1981 const frame_size = frame_allocs_slice.items(.abi_size);
1982 const frame_align = frame_allocs_slice.items(.abi_align);
1983
1984 const stack_frame_align = &frame_align[@intFromEnum(FrameIndex.stack_frame)];
1985 stack_frame_align.* = stack_frame_align.max(alloc.abi_align);
1986
1987 for (func.free_frame_indices.keys(), 0..) |frame_index, free_i| {
1988 const abi_size = frame_size[@intFromEnum(frame_index)];
1989 if (abi_size != alloc.abi_size) continue;
1990 const abi_align = &frame_align[@intFromEnum(frame_index)];
1991 abi_align.* = abi_align.max(alloc.abi_align);
1992
1993 _ = func.free_frame_indices.swapRemoveAt(free_i);
1994 return frame_index;
1995 }
1996 const frame_index: FrameIndex = @enumFromInt(func.frame_allocs.len);
1997 try func.frame_allocs.append(func.gpa, alloc);
1998 log.debug("allocated frame {}", .{frame_index});
1999 return frame_index;
2000}
2001
2002/// Use a pointer instruction as the basis for allocating stack memory.
2003fn allocMemPtr(func: *Func, inst: Air.Inst.Index) !FrameIndex {
2004 const pt = func.pt;
2005 const zcu = pt.zcu;
2006 const ptr_ty = func.typeOfIndex(inst);
2007 const val_ty = ptr_ty.childType(zcu);
2008 return func.allocFrameIndex(FrameAlloc.init(.{
2009 .size = math.cast(u32, val_ty.abiSize(zcu)) orelse {
2010 return func.fail("type '{f}' too big to fit into stack frame", .{val_ty.fmt(pt)});
2011 },
2012 .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"),
2013 }));
2014}
2015
2016fn typeRegClass(func: *Func, ty: Type) abi.RegisterClass {
2017 const pt = func.pt;
2018 const zcu = pt.zcu;
2019 return switch (ty.zigTypeTag(zcu)) {
2020 .float => .float,
2021 .vector => .vector,
2022 else => .int,
2023 };
2024}
2025
2026fn regGeneralClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet {
2027 return switch (ty.zigTypeTag(func.pt.zcu)) {
2028 .float => abi.Registers.Float.general_purpose,
2029 .vector => abi.Registers.Vector.general_purpose,
2030 else => abi.Registers.Integer.general_purpose,
2031 };
2032}
2033
2034fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet {
2035 return switch (ty.zigTypeTag(func.pt.zcu)) {
2036 .float => abi.Registers.Float.temporary,
2037 .vector => abi.Registers.Vector.general_purpose, // there are no temporary vector registers
2038 else => abi.Registers.Integer.temporary,
2039 };
2040}
2041
2042fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
2043 const pt = func.pt;
2044 const zcu = pt.zcu;
2045
2046 const bit_size = elem_ty.bitSize(zcu);
2047 const min_size: u64 = switch (elem_ty.zigTypeTag(zcu)) {
2048 .float => if (func.hasFeature(.d)) 64 else 32,
2049 .vector => 256, // TODO: calculate it from avl * vsew
2050 else => 64,
2051 };
2052
2053 if (reg_ok and bit_size <= min_size) {
2054 if (func.register_manager.tryAllocReg(inst, func.regGeneralClassForType(elem_ty))) |reg| {
2055 return .{ .register = reg };
2056 }
2057 } else if (reg_ok and elem_ty.zigTypeTag(zcu) == .vector) {
2058 return func.fail("did you forget to extend vector registers before allocating", .{});
2059 }
2060
2061 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(elem_ty, zcu));
2062 return .{ .load_frame = .{ .index = frame_index } };
2063}
2064
2065/// Allocates a register from the general purpose set and returns the Register and the Lock.
2066///
2067/// Up to the caller to unlock the register later.
2068fn allocReg(func: *Func, reg_class: abi.RegisterClass) !struct { Register, RegisterLock } {
2069 if (reg_class == .float and !func.hasFeature(.f))
2070 std.debug.panic("allocReg class == float where F isn't enabled", .{});
2071 if (reg_class == .vector and !func.hasFeature(.v))
2072 std.debug.panic("allocReg class == vector where V isn't enabled", .{});
2073
2074 const class = switch (reg_class) {
2075 .int => abi.Registers.Integer.general_purpose,
2076 .float => abi.Registers.Float.general_purpose,
2077 .vector => abi.Registers.Vector.general_purpose,
2078 };
2079
2080 const reg = try func.register_manager.allocReg(null, class);
2081 const lock = func.register_manager.lockRegAssumeUnused(reg);
2082 return .{ reg, lock };
2083}
2084
2085/// Similar to `allocReg` but will copy the MCValue into the Register unless `operand` is already
2086/// a register, in which case it will return a possible lock to that register.
2087fn promoteReg(func: *Func, ty: Type, operand: MCValue) !struct { Register, ?RegisterLock } {
2088 if (operand == .register) {
2089 const op_reg = operand.register;
2090 return .{ op_reg, func.register_manager.lockReg(operand.register) };
2091 }
2092
2093 const class = func.typeRegClass(ty);
2094 const reg, const lock = try func.allocReg(class);
2095 try func.genSetReg(ty, reg, operand);
2096 return .{ reg, lock };
2097}
2098
2099fn elemOffset(func: *Func, index_ty: Type, index: MCValue, elem_size: u64) !Register {
2100 const reg: Register = blk: {
2101 switch (index) {
2102 .immediate => |imm| {
2103 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
2104 // and set the register directly to the scaled offset as an immediate.
2105 const reg = try func.register_manager.allocReg(null, func.regGeneralClassForType(index_ty));
2106 try func.genSetReg(index_ty, reg, .{ .immediate = imm * elem_size });
2107 break :blk reg;
2108 },
2109 else => {
2110 const reg = try func.copyToTmpRegister(index_ty, index);
2111 const lock = func.register_manager.lockRegAssumeUnused(reg);
2112 defer func.register_manager.unlockReg(lock);
2113
2114 const result_reg, const result_lock = try func.allocReg(.int);
2115 defer func.register_manager.unlockReg(result_lock);
2116
2117 try func.genBinOp(
2118 .mul,
2119 .{ .register = reg },
2120 index_ty,
2121 .{ .immediate = elem_size },
2122 index_ty,
2123 result_reg,
2124 );
2125
2126 break :blk result_reg;
2127 },
2128 }
2129 };
2130 return reg;
2131}
2132
2133pub fn spillInstruction(func: *Func, reg: Register, inst: Air.Inst.Index) !void {
2134 const tracking = func.inst_tracking.getPtr(inst) orelse return;
2135 for (tracking.getRegs()) |tracked_reg| {
2136 if (tracked_reg.id() == reg.id()) break;
2137 } else unreachable; // spilled reg not tracked with spilled instruciton
2138 try tracking.spill(func, inst);
2139 try tracking.trackSpill(func, inst);
2140}
2141
2142pub fn spillRegisters(func: *Func, comptime registers: []const Register) !void {
2143 inline for (registers) |reg| try func.register_manager.getKnownReg(reg, null);
2144}
2145
2146/// Copies a value to a register without tracking the register. The register is not considered
2147/// allocated. A second call to `copyToTmpRegister` may return the same register.
2148/// This can have a side effect of spilling instructions to the stack to free up a register.
2149fn copyToTmpRegister(func: *Func, ty: Type, mcv: MCValue) !Register {
2150 log.debug("copyToTmpRegister ty: {f}", .{ty.fmt(func.pt)});
2151 const reg = try func.register_manager.allocReg(null, func.regTempClassForType(ty));
2152 try func.genSetReg(ty, reg, mcv);
2153 return reg;
2154}
2155
2156/// Allocates a new register and copies `mcv` into it.
2157/// `reg_owner` is the instruction that gets associated with the register in the register table.
2158/// This can have a side effect of spilling instructions to the stack to free up a register.
2159fn copyToNewRegister(func: *Func, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue {
2160 const ty = func.typeOfIndex(reg_owner);
2161 const reg = try func.register_manager.allocReg(reg_owner, func.regGeneralClassForType(ty));
2162 try func.genSetReg(func.typeOfIndex(reg_owner), reg, mcv);
2163 return MCValue{ .register = reg };
2164}
2165
2166fn airAlloc(func: *Func, inst: Air.Inst.Index) !void {
2167 const result = MCValue{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } };
2168 return func.finishAir(inst, result, .{ .none, .none, .none });
2169}
2170
2171fn airRetPtr(func: *Func, inst: Air.Inst.Index) !void {
2172 const result: MCValue = switch (func.ret_mcv.long) {
2173 .none => .{ .lea_frame = .{ .index = try func.allocMemPtr(inst) } },
2174 .load_frame => .{ .register_offset = .{
2175 .reg = (try func.copyToNewRegister(
2176 inst,
2177 func.ret_mcv.long,
2178 )).register,
2179 .off = func.ret_mcv.short.indirect.off,
2180 } },
2181 else => |t| return func.fail("TODO: airRetPtr {s}", .{@tagName(t)}),
2182 };
2183 return func.finishAir(inst, result, .{ .none, .none, .none });
2184}
2185
2186fn airFptrunc(func: *Func, inst: Air.Inst.Index) !void {
2187 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2188 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFptrunc for {}", .{func.target.cpu.arch});
2189 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2190}
2191
2192fn airFpext(func: *Func, inst: Air.Inst.Index) !void {
2193 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2194 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airFpext for {}", .{func.target.cpu.arch});
2195 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2196}
2197
2198fn airIntCast(func: *Func, inst: Air.Inst.Index) !void {
2199 const pt = func.pt;
2200 const zcu = pt.zcu;
2201 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2202 const src_ty = func.typeOf(ty_op.operand);
2203 const dst_ty = func.typeOfIndex(inst);
2204
2205 const result: MCValue = result: {
2206 const src_int_info = src_ty.intInfo(zcu);
2207 const dst_int_info = dst_ty.intInfo(zcu);
2208
2209 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
2210
2211 const src_mcv = try func.resolveInst(ty_op.operand);
2212
2213 const src_storage_bits: u16 = switch (src_mcv) {
2214 .register => 64,
2215 .load_frame => src_int_info.bits,
2216 else => return func.fail("airIntCast from {s}", .{@tagName(src_mcv)}),
2217 };
2218
2219 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2220 math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
2221 math.divCeil(u32, src_storage_bits, 64) catch unreachable and
2222 func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2223 const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true);
2224 try func.genCopy(min_ty, dst_mcv, src_mcv);
2225 break :dst dst_mcv;
2226 };
2227
2228 if (dst_int_info.bits <= src_int_info.bits)
2229 break :result dst_mcv;
2230
2231 if (dst_int_info.bits > 64 or src_int_info.bits > 64)
2232 break :result null; // TODO
2233
2234 break :result dst_mcv;
2235 } orelse return func.fail("TODO: implement airIntCast from {f} to {f}", .{
2236 src_ty.fmt(pt), dst_ty.fmt(pt),
2237 });
2238
2239 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2240}
2241
2242fn airTrunc(func: *Func, inst: Air.Inst.Index) !void {
2243 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2244 if (func.liveness.isUnused(inst))
2245 return func.finishAir(inst, .unreach, .{ ty_op.operand, .none, .none });
2246 // we assume no zeroext in the "Zig ABI", so it's fine to just not truncate it.
2247 const operand = try func.resolveInst(ty_op.operand);
2248
2249 // we can do it just to be safe, but this shouldn't be needed for no-runtime safety modes
2250 switch (operand) {
2251 .register => |reg| try func.truncateRegister(func.typeOf(ty_op.operand), reg),
2252 else => {},
2253 }
2254
2255 return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
2256}
2257
2258fn airNot(func: *Func, inst: Air.Inst.Index) !void {
2259 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2260 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
2261 const pt = func.pt;
2262 const zcu = pt.zcu;
2263
2264 const operand = try func.resolveInst(ty_op.operand);
2265 const ty = func.typeOf(ty_op.operand);
2266
2267 const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
2268 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
2269
2270 const dst_reg: Register =
2271 if (func.reuseOperand(inst, ty_op.operand, 0, operand) and operand == .register)
2272 operand.register
2273 else
2274 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
2275
2276 switch (ty.zigTypeTag(zcu)) {
2277 .bool => {
2278 _ = try func.addInst(.{
2279 .tag = .pseudo_not,
2280 .data = .{
2281 .rr = .{
2282 .rs = operand_reg,
2283 .rd = dst_reg,
2284 },
2285 },
2286 });
2287 },
2288 .int => {
2289 const size = ty.bitSize(zcu);
2290 if (!math.isPowerOfTwo(size))
2291 return func.fail("TODO: airNot non-pow 2 int size", .{});
2292
2293 switch (size) {
2294 32, 64 => {
2295 _ = try func.addInst(.{
2296 .tag = .xori,
2297 .data = .{
2298 .i_type = .{
2299 .rd = dst_reg,
2300 .rs1 = operand_reg,
2301 .imm12 = Immediate.s(-1),
2302 },
2303 },
2304 });
2305 },
2306 8, 16 => return func.fail("TODO: airNot 8 or 16, {}", .{size}),
2307 else => unreachable,
2308 }
2309 },
2310 else => unreachable,
2311 }
2312
2313 break :result .{ .register = dst_reg };
2314 };
2315 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2316}
2317
2318fn airSlice(func: *Func, inst: Air.Inst.Index) !void {
2319 const pt = func.pt;
2320 const zcu = pt.zcu;
2321 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2322 const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data;
2323
2324 const slice_ty = func.typeOfIndex(inst);
2325 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
2326
2327 const ptr_ty = func.typeOf(bin_op.lhs);
2328 try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs });
2329
2330 const len_ty = func.typeOf(bin_op.rhs);
2331 try func.genSetMem(
2332 .{ .frame = frame_index },
2333 @intCast(ptr_ty.abiSize(zcu)),
2334 len_ty,
2335 .{ .air_ref = bin_op.rhs },
2336 );
2337
2338 const result = MCValue{ .load_frame = .{ .index = frame_index } };
2339 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
2340}
2341
2342fn airBinOp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
2343 const pt = func.pt;
2344 const zcu = pt.zcu;
2345 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2346 const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
2347
2348 const dst_ty = func.typeOfIndex(inst);
2349 if (dst_ty.isAbiInt(zcu)) {
2350 const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
2351 const bit_size: u32 = @intCast(dst_ty.bitSize(zcu));
2352 if (abi_size * 8 > bit_size) {
2353 const dst_lock = switch (dst_mcv) {
2354 .register => |dst_reg| func.register_manager.lockRegAssumeUnused(dst_reg),
2355 else => null,
2356 };
2357 defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
2358
2359 if (dst_mcv.isRegister()) {
2360 try func.truncateRegister(dst_ty, dst_mcv.getReg().?);
2361 } else {
2362 const tmp_reg, const tmp_lock = try func.allocReg(.int);
2363 defer func.register_manager.unlockReg(tmp_lock);
2364
2365 const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1));
2366 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
2367 try func.genSetReg(hi_ty, tmp_reg, hi_mcv);
2368 try func.truncateRegister(dst_ty, tmp_reg);
2369 try func.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
2370 }
2371 }
2372 }
2373
2374 return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2375}
2376
2377fn binOp(
2378 func: *Func,
2379 maybe_inst: ?Air.Inst.Index,
2380 air_tag: Air.Inst.Tag,
2381 lhs_air: Air.Inst.Ref,
2382 rhs_air: Air.Inst.Ref,
2383) !MCValue {
2384 _ = maybe_inst;
2385 const pt = func.pt;
2386 const zcu = pt.zcu;
2387 const lhs_ty = func.typeOf(lhs_air);
2388 const rhs_ty = func.typeOf(rhs_air);
2389
2390 if (lhs_ty.isRuntimeFloat()) libcall: {
2391 const float_bits = lhs_ty.floatBits(func.target);
2392 const type_needs_libcall = switch (float_bits) {
2393 16 => true,
2394 32, 64 => false,
2395 80, 128 => true,
2396 else => unreachable,
2397 };
2398 if (!type_needs_libcall) break :libcall;
2399 return func.fail("binOp libcall runtime-float ops", .{});
2400 }
2401
2402 // don't have support for certain sizes of addition
2403 switch (lhs_ty.zigTypeTag(zcu)) {
2404 .vector => {}, // works differently and fails in a different place
2405 else => if (lhs_ty.bitSize(zcu) > 64) return func.fail("TODO: binOp >= 64 bits", .{}),
2406 }
2407
2408 const lhs_mcv = try func.resolveInst(lhs_air);
2409 const rhs_mcv = try func.resolveInst(rhs_air);
2410
2411 const class_for_dst_ty: abi.RegisterClass = switch (air_tag) {
2412 // will always return int register no matter the input
2413 .cmp_eq,
2414 .cmp_neq,
2415 .cmp_lt,
2416 .cmp_lte,
2417 .cmp_gt,
2418 .cmp_gte,
2419 => .int,
2420
2421 else => func.typeRegClass(lhs_ty),
2422 };
2423
2424 const dst_reg, const dst_lock = try func.allocReg(class_for_dst_ty);
2425 defer func.register_manager.unlockReg(dst_lock);
2426
2427 try func.genBinOp(
2428 air_tag,
2429 lhs_mcv,
2430 lhs_ty,
2431 rhs_mcv,
2432 rhs_ty,
2433 dst_reg,
2434 );
2435
2436 return .{ .register = dst_reg };
2437}
2438
2439/// Does the same thing as binOp however is meant to be used internally to the backend.
2440///
2441/// The `dst_reg` argument is meant to be caller-locked. Asserts that the binOp result can be
2442/// fit into the register.
2443///
2444/// Assumes that the `dst_reg` class is correct.
2445fn genBinOp(
2446 func: *Func,
2447 tag: Air.Inst.Tag,
2448 lhs_mcv: MCValue,
2449 lhs_ty: Type,
2450 rhs_mcv: MCValue,
2451 rhs_ty: Type,
2452 dst_reg: Register,
2453) !void {
2454 const pt = func.pt;
2455 const zcu = pt.zcu;
2456 const bit_size = lhs_ty.bitSize(zcu);
2457
2458 const is_unsigned = lhs_ty.isUnsignedInt(zcu);
2459
2460 const lhs_reg, const maybe_lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv);
2461 const rhs_reg, const maybe_rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv);
2462
2463 defer if (maybe_lhs_lock) |lock| func.register_manager.unlockReg(lock);
2464 defer if (maybe_rhs_lock) |lock| func.register_manager.unlockReg(lock);
2465
2466 switch (tag) {
2467 .add,
2468 .add_wrap,
2469 .sub,
2470 .sub_wrap,
2471 .mul,
2472 .mul_wrap,
2473 .rem,
2474 .div_trunc,
2475 .div_exact,
2476 => {
2477 switch (tag) {
2478 .rem,
2479 .div_trunc,
2480 .div_exact,
2481 => {
2482 if (!math.isPowerOfTwo(bit_size)) {
2483 try func.truncateRegister(lhs_ty, lhs_reg);
2484 try func.truncateRegister(rhs_ty, rhs_reg);
2485 }
2486 },
2487 else => {
2488 if (!math.isPowerOfTwo(bit_size))
2489 return func.fail(
2490 "TODO: genBinOp verify if needs to truncate {s} non-pow 2, found {}",
2491 .{ @tagName(tag), bit_size },
2492 );
2493 },
2494 }
2495
2496 switch (lhs_ty.zigTypeTag(zcu)) {
2497 .int => {
2498 const mnem: Mnemonic = switch (tag) {
2499 .add, .add_wrap => switch (bit_size) {
2500 8, 16, 64 => .add,
2501 32 => .addw,
2502 else => unreachable,
2503 },
2504 .sub, .sub_wrap => switch (bit_size) {
2505 8, 16, 32 => .subw,
2506 64 => .sub,
2507 else => unreachable,
2508 },
2509 .mul, .mul_wrap => switch (bit_size) {
2510 8, 16, 64 => .mul,
2511 32 => .mulw,
2512 else => unreachable,
2513 },
2514 .rem => switch (bit_size) {
2515 8, 16, 32 => if (is_unsigned) .remuw else .remw,
2516 else => if (is_unsigned) .remu else .rem,
2517 },
2518 .div_trunc, .div_exact => switch (bit_size) {
2519 8, 16, 32 => if (is_unsigned) .divuw else .divw,
2520 else => if (is_unsigned) .divu else .div,
2521 },
2522 else => unreachable,
2523 };
2524
2525 _ = try func.addInst(.{
2526 .tag = mnem,
2527 .data = .{
2528 .r_type = .{
2529 .rd = dst_reg,
2530 .rs1 = lhs_reg,
2531 .rs2 = rhs_reg,
2532 },
2533 },
2534 });
2535 },
2536 .float => {
2537 const mir_tag: Mnemonic = switch (tag) {
2538 .add => switch (bit_size) {
2539 32 => .fadds,
2540 64 => .faddd,
2541 else => unreachable,
2542 },
2543 .sub => switch (bit_size) {
2544 32 => .fsubs,
2545 64 => .fsubd,
2546 else => unreachable,
2547 },
2548 .mul => switch (bit_size) {
2549 32 => .fmuls,
2550 64 => .fmuld,
2551 else => unreachable,
2552 },
2553 else => return func.fail("TODO: genBinOp {s} Float", .{@tagName(tag)}),
2554 };
2555
2556 _ = try func.addInst(.{
2557 .tag = mir_tag,
2558 .data = .{
2559 .r_type = .{
2560 .rd = dst_reg,
2561 .rs1 = lhs_reg,
2562 .rs2 = rhs_reg,
2563 },
2564 },
2565 });
2566 },
2567 .vector => {
2568 const num_elem = lhs_ty.vectorLen(zcu);
2569 const elem_size = lhs_ty.childType(zcu).bitSize(zcu);
2570
2571 const child_ty = lhs_ty.childType(zcu);
2572
2573 const mir_tag: Mnemonic = switch (tag) {
2574 .add => switch (child_ty.zigTypeTag(zcu)) {
2575 .int => .vaddvv,
2576 .float => .vfaddvv,
2577 else => unreachable,
2578 },
2579 .sub => switch (child_ty.zigTypeTag(zcu)) {
2580 .int => .vsubvv,
2581 .float => .vfsubvv,
2582 else => unreachable,
2583 },
2584 .mul => switch (child_ty.zigTypeTag(zcu)) {
2585 .int => .vmulvv,
2586 .float => .vfmulvv,
2587 else => unreachable,
2588 },
2589 else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}),
2590 };
2591
2592 try func.setVl(.zero, num_elem, .{
2593 .vsew = switch (elem_size) {
2594 8 => .@"8",
2595 16 => .@"16",
2596 32 => .@"32",
2597 64 => .@"64",
2598 else => return func.fail("TODO: genBinOp > 64 bit elements, found {d}", .{elem_size}),
2599 },
2600 .vlmul = .m1,
2601 .vma = true,
2602 .vta = true,
2603 });
2604
2605 _ = try func.addInst(.{
2606 .tag = mir_tag,
2607 .data = .{
2608 .r_type = .{
2609 .rd = dst_reg,
2610 .rs1 = rhs_reg,
2611 .rs2 = lhs_reg,
2612 },
2613 },
2614 });
2615 },
2616 else => unreachable,
2617 }
2618 },
2619
2620 .add_sat,
2621 => {
2622 if (bit_size != 64 or !is_unsigned)
2623 return func.fail("TODO: genBinOp ty: {f}", .{lhs_ty.fmt(pt)});
2624
2625 const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg });
2626 const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg);
2627 defer func.register_manager.unlockReg(tmp_lock);
2628
2629 _ = try func.addInst(.{
2630 .tag = .add,
2631 .data = .{ .r_type = .{
2632 .rd = tmp_reg,
2633 .rs1 = rhs_reg,
2634 .rs2 = lhs_reg,
2635 } },
2636 });
2637
2638 _ = try func.addInst(.{
2639 .tag = .sltu,
2640 .data = .{ .r_type = .{
2641 .rd = dst_reg,
2642 .rs1 = tmp_reg,
2643 .rs2 = lhs_reg,
2644 } },
2645 });
2646
2647 // neg dst_reg, dst_reg
2648 _ = try func.addInst(.{
2649 .tag = .sub,
2650 .data = .{ .r_type = .{
2651 .rd = dst_reg,
2652 .rs1 = .zero,
2653 .rs2 = dst_reg,
2654 } },
2655 });
2656
2657 _ = try func.addInst(.{
2658 .tag = .@"or",
2659 .data = .{ .r_type = .{
2660 .rd = dst_reg,
2661 .rs1 = dst_reg,
2662 .rs2 = tmp_reg,
2663 } },
2664 });
2665 },
2666
2667 .ptr_add,
2668 .ptr_sub,
2669 => {
2670 const tmp_reg = try func.copyToTmpRegister(rhs_ty, .{ .register = rhs_reg });
2671 const tmp_mcv = MCValue{ .register = tmp_reg };
2672 const tmp_lock = func.register_manager.lockRegAssumeUnused(tmp_reg);
2673 defer func.register_manager.unlockReg(tmp_lock);
2674
2675 // RISC-V has no immediate mul, so we copy the size to a temporary register
2676 const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu);
2677 const elem_size_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = elem_size });
2678
2679 try func.genBinOp(
2680 .mul,
2681 tmp_mcv,
2682 rhs_ty,
2683 .{ .register = elem_size_reg },
2684 Type.u64,
2685 tmp_reg,
2686 );
2687
2688 try func.genBinOp(
2689 switch (tag) {
2690 .ptr_add => .add,
2691 .ptr_sub => .sub,
2692 else => unreachable,
2693 },
2694 lhs_mcv,
2695 Type.u64, // we know it's a pointer, so it'll be usize.
2696 tmp_mcv,
2697 Type.u64,
2698 dst_reg,
2699 );
2700 },
2701
2702 .bit_and,
2703 .bit_or,
2704 .bool_and,
2705 .bool_or,
2706 => {
2707 _ = try func.addInst(.{
2708 .tag = switch (tag) {
2709 .bit_and, .bool_and => .@"and",
2710 .bit_or, .bool_or => .@"or",
2711 else => unreachable,
2712 },
2713 .data = .{
2714 .r_type = .{
2715 .rd = dst_reg,
2716 .rs1 = lhs_reg,
2717 .rs2 = rhs_reg,
2718 },
2719 },
2720 });
2721
2722 switch (tag) {
2723 .bool_and,
2724 .bool_or,
2725 => try func.truncateRegister(Type.bool, dst_reg),
2726 else => {},
2727 }
2728 },
2729
2730 .shr,
2731 .shr_exact,
2732 .shl,
2733 .shl_exact,
2734 => {
2735 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) return func.fail("TODO: vector shift with scalar rhs", .{});
2736 if (bit_size > 64) return func.fail("TODO: genBinOp shift > 64 bits, {}", .{bit_size});
2737 try func.truncateRegister(rhs_ty, rhs_reg);
2738
2739 const mir_tag: Mnemonic = switch (tag) {
2740 .shl, .shl_exact => switch (bit_size) {
2741 1...31, 33...64 => .sll,
2742 32 => .sllw,
2743 else => unreachable,
2744 },
2745 .shr, .shr_exact => switch (bit_size) {
2746 1...31, 33...64 => .srl,
2747 32 => .srlw,
2748 else => unreachable,
2749 },
2750 else => unreachable,
2751 };
2752
2753 _ = try func.addInst(.{
2754 .tag = mir_tag,
2755 .data = .{ .r_type = .{
2756 .rd = dst_reg,
2757 .rs1 = lhs_reg,
2758 .rs2 = rhs_reg,
2759 } },
2760 });
2761 },
2762
2763 // TODO: move the isel logic out of lower and into here.
2764 .cmp_eq,
2765 .cmp_neq,
2766 .cmp_lt,
2767 .cmp_lte,
2768 .cmp_gt,
2769 .cmp_gte,
2770 => {
2771 assert(lhs_reg.class() == rhs_reg.class());
2772 if (lhs_reg.class() == .int) {
2773 try func.truncateRegister(lhs_ty, lhs_reg);
2774 try func.truncateRegister(rhs_ty, rhs_reg);
2775 }
2776
2777 _ = try func.addInst(.{
2778 .tag = .pseudo_compare,
2779 .data = .{
2780 .compare = .{
2781 .op = switch (tag) {
2782 .cmp_eq => .eq,
2783 .cmp_neq => .neq,
2784 .cmp_lt => .lt,
2785 .cmp_lte => .lte,
2786 .cmp_gt => .gt,
2787 .cmp_gte => .gte,
2788 else => unreachable,
2789 },
2790 .rd = dst_reg,
2791 .rs1 = lhs_reg,
2792 .rs2 = rhs_reg,
2793 .ty = lhs_ty,
2794 },
2795 },
2796 });
2797 },
2798
2799 // A branchless @min/@max sequence.
2800 //
2801 // Assume that a0 and a1 are the lhs and rhs respectively.
2802 // Also assume that a2 is the destination register.
2803 //
2804 // Algorithm:
2805 // slt s0, a0, a1
2806 // sub s0, zero, s0
2807 // xor a2, a0, a1
2808 // and s0, a2, s0
2809 // xor a2, a0, s0 # a0 is @min, a1 is @max
2810 //
2811 // "slt s0, a0, a1" will set s0 to 1 if a0 is less than a1, and 1 otherwise.
2812 //
2813 // "sub s0, zero, s0" will set all the bits of s0 to 1 if it was 1, otherwise it'll remain at 0.
2814 //
2815 // "xor a2, a0, a1" stores the bitwise XOR of a0 and a1 in a2. Effectively getting the difference between them.
2816 //
2817 // "and a0, a2, s0" here we mask the result of the XOR with the negated s0. If a0 < a1, s0 is -1, which
2818 // doesn't change the bits of a2. If a0 >= a1, s0 is 0, nullifying a2.
2819 //
2820 // "xor a2, a0, s0" the final XOR operation adjusts a2 to be the minimum value of a0 and a1. If a0 was less than
2821 // a1, s0 was -1, flipping all the bits in a2 and effectively restoring a0. If a0 was greater than or equal to a1,
2822 // s0 was 0, leaving a2 unchanged as a0.
2823 .min, .max => {
2824 switch (lhs_ty.zigTypeTag(zcu)) {
2825 .int => {
2826 const int_info = lhs_ty.intInfo(zcu);
2827
2828 const mask_reg, const mask_lock = try func.allocReg(.int);
2829 defer func.register_manager.unlockReg(mask_lock);
2830
2831 _ = try func.addInst(.{
2832 .tag = if (int_info.signedness == .unsigned) .sltu else .slt,
2833 .data = .{ .r_type = .{
2834 .rd = mask_reg,
2835 .rs1 = lhs_reg,
2836 .rs2 = rhs_reg,
2837 } },
2838 });
2839
2840 _ = try func.addInst(.{
2841 .tag = .sub,
2842 .data = .{ .r_type = .{
2843 .rd = mask_reg,
2844 .rs1 = .zero,
2845 .rs2 = mask_reg,
2846 } },
2847 });
2848
2849 _ = try func.addInst(.{
2850 .tag = .xor,
2851 .data = .{ .r_type = .{
2852 .rd = dst_reg,
2853 .rs1 = lhs_reg,
2854 .rs2 = rhs_reg,
2855 } },
2856 });
2857
2858 _ = try func.addInst(.{
2859 .tag = .@"and",
2860 .data = .{ .r_type = .{
2861 .rd = mask_reg,
2862 .rs1 = dst_reg,
2863 .rs2 = mask_reg,
2864 } },
2865 });
2866
2867 _ = try func.addInst(.{
2868 .tag = .xor,
2869 .data = .{ .r_type = .{
2870 .rd = dst_reg,
2871 .rs1 = if (tag == .min) rhs_reg else lhs_reg,
2872 .rs2 = mask_reg,
2873 } },
2874 });
2875 },
2876 else => |t| return func.fail("TODO: genBinOp min/max for {s}", .{@tagName(t)}),
2877 }
2878 },
2879 else => return func.fail("TODO: genBinOp {}", .{tag}),
2880 }
2881}
2882
2883fn airPtrArithmetic(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
2884 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2885 const bin_op = func.air.extraData(Air.Bin, ty_pl.payload).data;
2886 const dst_mcv = try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
2887 return func.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2888}
2889
2890fn airAddWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
2891 const pt = func.pt;
2892 const zcu = pt.zcu;
2893 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2894 const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
2895
2896 const rhs_ty = func.typeOf(extra.rhs);
2897 const lhs_ty = func.typeOf(extra.lhs);
2898
2899 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
2900 switch (lhs_ty.zigTypeTag(zcu)) {
2901 .vector => return func.fail("TODO implement add with overflow for Vector type", .{}),
2902 .int => {
2903 const int_info = lhs_ty.intInfo(zcu);
2904
2905 const tuple_ty = func.typeOfIndex(inst);
2906 const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
2907 const offset = result_mcv.load_frame;
2908
2909 if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) {
2910 const add_result = try func.binOp(null, .add, extra.lhs, extra.rhs);
2911
2912 try func.genSetMem(
2913 .{ .frame = offset.index },
2914 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
2915 lhs_ty,
2916 add_result,
2917 );
2918
2919 const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result);
2920 const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
2921 defer func.register_manager.unlockReg(trunc_reg_lock);
2922
2923 const overflow_reg, const overflow_lock = try func.allocReg(.int);
2924 defer func.register_manager.unlockReg(overflow_lock);
2925
2926 // if the result isn't equal after truncating it to the given type,
2927 // an overflow must have happened.
2928 try func.truncateRegister(lhs_ty, trunc_reg);
2929 try func.genBinOp(
2930 .cmp_neq,
2931 add_result,
2932 lhs_ty,
2933 .{ .register = trunc_reg },
2934 rhs_ty,
2935 overflow_reg,
2936 );
2937
2938 try func.genSetMem(
2939 .{ .frame = offset.index },
2940 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
2941 Type.u1,
2942 .{ .register = overflow_reg },
2943 );
2944
2945 break :result result_mcv;
2946 } else {
2947 const rhs_mcv = try func.resolveInst(extra.rhs);
2948 const lhs_mcv = try func.resolveInst(extra.lhs);
2949
2950 const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs_mcv);
2951 const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs_mcv);
2952 defer {
2953 if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
2954 if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
2955 }
2956
2957 try func.truncateRegister(rhs_ty, rhs_reg);
2958 try func.truncateRegister(lhs_ty, lhs_reg);
2959
2960 const dest_reg, const dest_lock = try func.allocReg(.int);
2961 defer func.register_manager.unlockReg(dest_lock);
2962
2963 _ = try func.addInst(.{
2964 .tag = .add,
2965 .data = .{ .r_type = .{
2966 .rs1 = rhs_reg,
2967 .rs2 = lhs_reg,
2968 .rd = dest_reg,
2969 } },
2970 });
2971
2972 try func.truncateRegister(func.typeOfIndex(inst), dest_reg);
2973 const add_result: MCValue = .{ .register = dest_reg };
2974
2975 try func.genSetMem(
2976 .{ .frame = offset.index },
2977 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
2978 lhs_ty,
2979 add_result,
2980 );
2981
2982 const trunc_reg = try func.copyToTmpRegister(lhs_ty, add_result);
2983 const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
2984 defer func.register_manager.unlockReg(trunc_reg_lock);
2985
2986 const overflow_reg, const overflow_lock = try func.allocReg(.int);
2987 defer func.register_manager.unlockReg(overflow_lock);
2988
2989 // if the result isn't equal after truncating it to the given type,
2990 // an overflow must have happened.
2991 try func.truncateRegister(lhs_ty, trunc_reg);
2992 try func.genBinOp(
2993 .cmp_neq,
2994 add_result,
2995 lhs_ty,
2996 .{ .register = trunc_reg },
2997 rhs_ty,
2998 overflow_reg,
2999 );
3000
3001 try func.genSetMem(
3002 .{ .frame = offset.index },
3003 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3004 Type.u1,
3005 .{ .register = overflow_reg },
3006 );
3007
3008 break :result result_mcv;
3009 }
3010 },
3011 else => unreachable,
3012 }
3013 };
3014
3015 return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3016}
3017
3018fn airSubWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3019 const pt = func.pt;
3020 const zcu = pt.zcu;
3021 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3022 const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3023
3024 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3025 const lhs = try func.resolveInst(extra.lhs);
3026 const rhs = try func.resolveInst(extra.rhs);
3027 const lhs_ty = func.typeOf(extra.lhs);
3028 const rhs_ty = func.typeOf(extra.rhs);
3029
3030 const int_info = lhs_ty.intInfo(zcu);
3031
3032 if (!math.isPowerOfTwo(int_info.bits) or int_info.bits < 8) {
3033 return func.fail("TODO: airSubWithOverflow non-power of 2 and less than 8 bits", .{});
3034 }
3035
3036 if (int_info.bits > 64) {
3037 return func.fail("TODO: airSubWithOverflow > 64 bits", .{});
3038 }
3039
3040 const tuple_ty = func.typeOfIndex(inst);
3041 const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
3042 const offset = result_mcv.load_frame;
3043
3044 const dest_mcv = try func.binOp(null, .sub, extra.lhs, extra.rhs);
3045 assert(dest_mcv == .register);
3046 const dest_reg = dest_mcv.register;
3047
3048 try func.genSetMem(
3049 .{ .frame = offset.index },
3050 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
3051 lhs_ty,
3052 .{ .register = dest_reg },
3053 );
3054
3055 const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs);
3056 defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
3057
3058 const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs);
3059 defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
3060
3061 const overflow_reg = try func.copyToTmpRegister(Type.u64, .{ .immediate = 0 });
3062
3063 const overflow_lock = func.register_manager.lockRegAssumeUnused(overflow_reg);
3064 defer func.register_manager.unlockReg(overflow_lock);
3065
3066 switch (int_info.signedness) {
3067 .unsigned => {
3068 _ = try func.addInst(.{
3069 .tag = .sltu,
3070 .data = .{ .r_type = .{
3071 .rd = overflow_reg,
3072 .rs1 = lhs_reg,
3073 .rs2 = rhs_reg,
3074 } },
3075 });
3076
3077 try func.genSetMem(
3078 .{ .frame = offset.index },
3079 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3080 Type.u1,
3081 .{ .register = overflow_reg },
3082 );
3083
3084 break :result result_mcv;
3085 },
3086 .signed => {
3087 switch (int_info.bits) {
3088 64 => {
3089 _ = try func.addInst(.{
3090 .tag = .slt,
3091 .data = .{ .r_type = .{
3092 .rd = overflow_reg,
3093 .rs1 = overflow_reg,
3094 .rs2 = rhs_reg,
3095 } },
3096 });
3097
3098 _ = try func.addInst(.{
3099 .tag = .slt,
3100 .data = .{ .r_type = .{
3101 .rd = rhs_reg,
3102 .rs1 = rhs_reg,
3103 .rs2 = lhs_reg,
3104 } },
3105 });
3106
3107 _ = try func.addInst(.{
3108 .tag = .xor,
3109 .data = .{ .r_type = .{
3110 .rd = lhs_reg,
3111 .rs1 = overflow_reg,
3112 .rs2 = rhs_reg,
3113 } },
3114 });
3115
3116 try func.genBinOp(
3117 .cmp_neq,
3118 .{ .register = overflow_reg },
3119 Type.u64,
3120 .{ .register = rhs_reg },
3121 Type.u64,
3122 overflow_reg,
3123 );
3124
3125 try func.genSetMem(
3126 .{ .frame = offset.index },
3127 offset.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
3128 Type.u1,
3129 .{ .register = overflow_reg },
3130 );
3131
3132 break :result result_mcv;
3133 },
3134 else => |int_bits| return func.fail("TODO: airSubWithOverflow signed {}", .{int_bits}),
3135 }
3136 },
3137 }
3138 };
3139
3140 return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3141}
3142
3143fn airMulWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3144 const pt = func.pt;
3145 const zcu = pt.zcu;
3146 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3147 const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3148
3149 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3150 const lhs = try func.resolveInst(extra.lhs);
3151 const rhs = try func.resolveInst(extra.rhs);
3152 const lhs_ty = func.typeOf(extra.lhs);
3153 const rhs_ty = func.typeOf(extra.rhs);
3154
3155 const tuple_ty = func.typeOfIndex(inst);
3156
3157 // genSetReg needs to support register_offset src_mcv for this to be true.
3158 const result_mcv = try func.allocRegOrMem(tuple_ty, inst, false);
3159
3160 const result_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
3161 const overflow_off: i32 = @intCast(tuple_ty.structFieldOffset(1, zcu));
3162
3163 const dest_reg, const dest_lock = try func.allocReg(.int);
3164 defer func.register_manager.unlockReg(dest_lock);
3165
3166 try func.genBinOp(
3167 .mul,
3168 lhs,
3169 lhs_ty,
3170 rhs,
3171 rhs_ty,
3172 dest_reg,
3173 );
3174
3175 try func.genCopy(
3176 lhs_ty,
3177 result_mcv.offset(result_off),
3178 .{ .register = dest_reg },
3179 );
3180
3181 switch (lhs_ty.zigTypeTag(zcu)) {
3182 else => |x| return func.fail("TODO: airMulWithOverflow {s}", .{@tagName(x)}),
3183 .int => {
3184 if (std.debug.runtime_safety) assert(lhs_ty.eql(rhs_ty, zcu));
3185
3186 const trunc_reg = try func.copyToTmpRegister(lhs_ty, .{ .register = dest_reg });
3187 const trunc_reg_lock = func.register_manager.lockRegAssumeUnused(trunc_reg);
3188 defer func.register_manager.unlockReg(trunc_reg_lock);
3189
3190 const overflow_reg, const overflow_lock = try func.allocReg(.int);
3191 defer func.register_manager.unlockReg(overflow_lock);
3192
3193 // if the result isn't equal after truncating it to the given type,
3194 // an overflow must have happened.
3195 try func.truncateRegister(func.typeOf(extra.lhs), trunc_reg);
3196 try func.genBinOp(
3197 .cmp_neq,
3198 .{ .register = dest_reg },
3199 lhs_ty,
3200 .{ .register = trunc_reg },
3201 rhs_ty,
3202 overflow_reg,
3203 );
3204
3205 try func.genCopy(
3206 lhs_ty,
3207 result_mcv.offset(overflow_off),
3208 .{ .register = overflow_reg },
3209 );
3210
3211 break :result result_mcv;
3212 },
3213 }
3214 };
3215
3216 return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
3217}
3218
3219fn airShlWithOverflow(func: *Func, inst: Air.Inst.Index) !void {
3220 const zcu = func.pt.zcu;
3221 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3222 const result: MCValue = if (func.liveness.isUnused(inst))
3223 .unreach
3224 else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu))
3225 return func.fail("TODO implement vector airShlWithOverflow with scalar rhs", .{})
3226 else
3227 return func.fail("TODO implement airShlWithOverflow", .{});
3228 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3229}
3230
3231fn airSubSat(func: *Func, inst: Air.Inst.Index) !void {
3232 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3233 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSubSat", .{});
3234 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3235}
3236
3237fn airMulSat(func: *Func, inst: Air.Inst.Index) !void {
3238 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3239 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airMulSat", .{});
3240 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3241}
3242
3243fn airShlSat(func: *Func, inst: Air.Inst.Index) !void {
3244 const zcu = func.pt.zcu;
3245 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3246 const result: MCValue = if (func.liveness.isUnused(inst))
3247 .unreach
3248 else if (func.typeOf(bin_op.lhs).isVector(zcu) and !func.typeOf(bin_op.rhs).isVector(zcu))
3249 return func.fail("TODO implement vector airShlSat with scalar rhs", .{})
3250 else
3251 return func.fail("TODO implement airShlSat", .{});
3252 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3253}
3254
3255fn airOptionalPayload(func: *Func, inst: Air.Inst.Index) !void {
3256 const zcu = func.pt.zcu;
3257 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3258 const result: MCValue = result: {
3259 const pl_ty = func.typeOfIndex(inst);
3260 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
3261
3262 const opt_mcv = try func.resolveInst(ty_op.operand);
3263 if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
3264 switch (opt_mcv) {
3265 .register => |pl_reg| try func.truncateRegister(pl_ty, pl_reg),
3266 else => {},
3267 }
3268 break :result opt_mcv;
3269 }
3270
3271 const pl_mcv = try func.allocRegOrMem(pl_ty, inst, true);
3272 try func.genCopy(pl_ty, pl_mcv, opt_mcv);
3273 break :result pl_mcv;
3274 };
3275 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3276}
3277
3278fn airOptionalPayloadPtr(func: *Func, inst: Air.Inst.Index) !void {
3279 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3280 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement .optional_payload_ptr for {}", .{func.target.cpu.arch});
3281 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3282}
3283
3284fn airOptionalPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void {
3285 const zcu = func.pt.zcu;
3286
3287 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3288 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3289 const dst_ty = func.typeOfIndex(inst);
3290 const src_ty = func.typeOf(ty_op.operand);
3291 const opt_ty = src_ty.childType(zcu);
3292 const src_mcv = try func.resolveInst(ty_op.operand);
3293
3294 if (opt_ty.optionalReprIsPayload(zcu)) {
3295 break :result if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3296 src_mcv
3297 else
3298 try func.copyToNewRegister(inst, src_mcv);
3299 }
3300
3301 const dst_mcv: MCValue = if (src_mcv.isRegister() and
3302 func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3303 src_mcv
3304 else
3305 try func.copyToNewRegister(inst, src_mcv);
3306
3307 const pl_ty = dst_ty.childType(zcu);
3308 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
3309 try func.genSetMem(
3310 .{ .reg = dst_mcv.getReg().? },
3311 pl_abi_size,
3312 Type.bool,
3313 .{ .immediate = 1 },
3314 );
3315 break :result dst_mcv;
3316 };
3317 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3318}
3319
3320fn airUnwrapErrErr(func: *Func, inst: Air.Inst.Index) !void {
3321 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3322 const pt = func.pt;
3323 const zcu = pt.zcu;
3324 const err_union_ty = func.typeOf(ty_op.operand);
3325 const err_ty = err_union_ty.errorUnionSet(zcu);
3326 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3327 const operand = try func.resolveInst(ty_op.operand);
3328
3329 const result: MCValue = result: {
3330 if (err_ty.errorSetIsEmpty(zcu)) {
3331 break :result .{ .immediate = 0 };
3332 }
3333
3334 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3335 break :result operand;
3336 }
3337
3338 const err_off: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
3339
3340 switch (operand) {
3341 .register => |reg| {
3342 const eu_lock = func.register_manager.lockReg(reg);
3343 defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
3344
3345 const result = try func.copyToNewRegister(inst, operand);
3346 if (err_off > 0) {
3347 try func.genBinOp(
3348 .shr,
3349 result,
3350 err_union_ty,
3351 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
3352 Type.u8,
3353 result.register,
3354 );
3355 }
3356 break :result result;
3357 },
3358 .load_frame => |frame_addr| break :result .{ .load_frame = .{
3359 .index = frame_addr.index,
3360 .off = frame_addr.off + @as(i32, @intCast(err_off)),
3361 } },
3362 else => return func.fail("TODO implement unwrap_err_err for {}", .{operand}),
3363 }
3364 };
3365
3366 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3367}
3368
3369fn airUnwrapErrPayload(func: *Func, inst: Air.Inst.Index) !void {
3370 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3371 const operand_ty = func.typeOf(ty_op.operand);
3372 const operand = try func.resolveInst(ty_op.operand);
3373 const result = try func.genUnwrapErrUnionPayloadMir(operand_ty, operand);
3374 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3375}
3376
3377fn genUnwrapErrUnionPayloadMir(
3378 func: *Func,
3379 err_union_ty: Type,
3380 err_union: MCValue,
3381) !MCValue {
3382 const pt = func.pt;
3383 const zcu = pt.zcu;
3384 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3385
3386 const result: MCValue = result: {
3387 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
3388
3389 const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
3390 switch (err_union) {
3391 .load_frame => |frame_addr| break :result .{ .load_frame = .{
3392 .index = frame_addr.index,
3393 .off = frame_addr.off + payload_off,
3394 } },
3395 .register => |reg| {
3396 const eu_lock = func.register_manager.lockReg(reg);
3397 defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
3398
3399 const result_reg = try func.copyToTmpRegister(err_union_ty, err_union);
3400 if (payload_off > 0) {
3401 try func.genBinOp(
3402 .shr,
3403 .{ .register = result_reg },
3404 err_union_ty,
3405 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
3406 Type.u8,
3407 result_reg,
3408 );
3409 }
3410 break :result .{ .register = result_reg };
3411 },
3412 else => return func.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
3413 }
3414 };
3415
3416 return result;
3417}
3418
3419// *(E!T) -> E
3420fn airUnwrapErrErrPtr(func: *Func, inst: Air.Inst.Index) !void {
3421 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3422 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union error ptr for {}", .{func.target.cpu.arch});
3423 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3424}
3425
3426// *(E!T) -> *T
3427fn airUnwrapErrPayloadPtr(func: *Func, inst: Air.Inst.Index) !void {
3428 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3429 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement unwrap error union payload ptr for {}", .{func.target.cpu.arch});
3430 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3431}
3432
3433// *(E!T) => *T
3434fn airErrUnionPayloadPtrSet(func: *Func, inst: Air.Inst.Index) !void {
3435 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3436 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3437 const zcu = func.pt.zcu;
3438 const src_ty = func.typeOf(ty_op.operand);
3439 const src_mcv = try func.resolveInst(ty_op.operand);
3440
3441 // `src_reg` contains the pointer to the error union
3442 const src_reg = switch (src_mcv) {
3443 .register => |reg| reg,
3444 else => try func.copyToTmpRegister(src_ty, src_mcv),
3445 };
3446 const src_lock = func.register_manager.lockRegAssumeUnused(src_reg);
3447 defer func.register_manager.unlockReg(src_lock);
3448
3449 // we set the place of where the error would have been to 0
3450 const eu_ty = src_ty.childType(zcu);
3451 const pl_ty = eu_ty.errorUnionPayload(zcu);
3452 const err_ty = eu_ty.errorUnionSet(zcu);
3453 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3454 try func.genSetMem(.{ .reg = src_reg }, err_off, err_ty, .{ .immediate = 0 });
3455
3456 const dst_reg, const dst_lock = if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv))
3457 .{ src_reg, null }
3458 else
3459 try func.allocReg(.int);
3460 defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
3461
3462 // move the pointer to be at the payload
3463 const pl_off = errUnionPayloadOffset(pl_ty, zcu);
3464 try func.genBinOp(
3465 .add,
3466 .{ .register = src_reg },
3467 Type.u64,
3468 .{ .immediate = pl_off },
3469 Type.u64,
3470 dst_reg,
3471 );
3472
3473 break :result .{ .register = dst_reg };
3474 };
3475 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3476}
3477
3478fn airErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void {
3479 const result: MCValue = if (func.liveness.isUnused(inst))
3480 .unreach
3481 else
3482 return func.fail("TODO implement airErrReturnTrace for {}", .{func.target.cpu.arch});
3483 return func.finishAir(inst, result, .{ .none, .none, .none });
3484}
3485
3486fn airSetErrReturnTrace(func: *Func, inst: Air.Inst.Index) !void {
3487 _ = inst;
3488 return func.fail("TODO implement airSetErrReturnTrace for {}", .{func.target.cpu.arch});
3489}
3490
3491fn airSaveErrReturnTraceIndex(func: *Func, inst: Air.Inst.Index) !void {
3492 _ = inst;
3493 return func.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{func.target.cpu.arch});
3494}
3495
3496fn airWrapOptional(func: *Func, inst: Air.Inst.Index) !void {
3497 const pt = func.pt;
3498 const zcu = pt.zcu;
3499 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3500 const result: MCValue = result: {
3501 const pl_ty = func.typeOf(ty_op.operand);
3502 if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 };
3503
3504 const opt_ty = func.typeOfIndex(inst);
3505 const pl_mcv = try func.resolveInst(ty_op.operand);
3506 const same_repr = opt_ty.optionalReprIsPayload(zcu);
3507 if (same_repr and func.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv;
3508
3509 const pl_lock: ?RegisterLock = switch (pl_mcv) {
3510 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3511 else => null,
3512 };
3513 defer if (pl_lock) |lock| func.register_manager.unlockReg(lock);
3514
3515 const opt_mcv = try func.allocRegOrMem(opt_ty, inst, false);
3516 try func.genCopy(pl_ty, opt_mcv, pl_mcv);
3517
3518 if (!same_repr) {
3519 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
3520 switch (opt_mcv) {
3521 .load_frame => |frame_addr| {
3522 try func.genCopy(pl_ty, opt_mcv, pl_mcv);
3523 try func.genSetMem(
3524 .{ .frame = frame_addr.index },
3525 frame_addr.off + pl_abi_size,
3526 Type.u8,
3527 .{ .immediate = 1 },
3528 );
3529 },
3530 else => unreachable,
3531 }
3532 }
3533 break :result opt_mcv;
3534 };
3535 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3536}
3537
3538/// T to E!T
3539fn airWrapErrUnionPayload(func: *Func, inst: Air.Inst.Index) !void {
3540 const pt = func.pt;
3541 const zcu = pt.zcu;
3542 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3543
3544 const eu_ty = ty_op.ty.toType();
3545 const pl_ty = eu_ty.errorUnionPayload(zcu);
3546 const err_ty = eu_ty.errorUnionSet(zcu);
3547 const operand = try func.resolveInst(ty_op.operand);
3548
3549 const result: MCValue = result: {
3550 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
3551
3552 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
3553 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
3554 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3555 try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
3556 try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 });
3557 break :result .{ .load_frame = .{ .index = frame_index } };
3558 };
3559
3560 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3561}
3562
3563/// E to E!T
3564fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void {
3565 const pt = func.pt;
3566 const zcu = pt.zcu;
3567 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3568
3569 const eu_ty = ty_op.ty.toType();
3570 const pl_ty = eu_ty.errorUnionPayload(zcu);
3571 const err_ty = eu_ty.errorUnionSet(zcu);
3572
3573 const result: MCValue = result: {
3574 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try func.resolveInst(ty_op.operand);
3575
3576 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu));
3577 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
3578 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
3579 try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null });
3580 const operand = try func.resolveInst(ty_op.operand);
3581 try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand);
3582 break :result .{ .load_frame = .{ .index = frame_index } };
3583 };
3584 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3585}
3586
3587fn airRuntimeNavPtr(func: *Func, inst: Air.Inst.Index) !void {
3588 const zcu = func.pt.zcu;
3589 const ip = &zcu.intern_pool;
3590 const ty_nav = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
3591 const ptr_ty: Type = .fromInterned(ty_nav.ty);
3592
3593 const nav = ip.getNav(ty_nav.nav);
3594 const tlv_sym_index = if (func.bin_file.cast(.elf)) |elf_file| sym: {
3595 const zo = elf_file.zigObjectPtr().?;
3596 if (nav.getExtern(ip)) |e| {
3597 break :sym try elf_file.getGlobalSymbol(nav.name.toSlice(ip), e.lib_name.toSlice(ip));
3598 }
3599 break :sym try zo.getOrCreateMetadataForNav(zcu, ty_nav.nav);
3600 } else return func.fail("TODO runtime_nav_ptr on {}", .{func.bin_file.tag});
3601
3602 const dest_mcv = try func.allocRegOrMem(ptr_ty, inst, true);
3603 if (dest_mcv.isRegister()) {
3604 _ = try func.addInst(.{
3605 .tag = .pseudo_load_tlv,
3606 .data = .{ .reloc = .{
3607 .register = dest_mcv.getReg().?,
3608 .atom_index = try func.owner.getSymbolIndex(func),
3609 .sym_index = tlv_sym_index,
3610 } },
3611 });
3612 } else {
3613 const tmp_reg, const tmp_lock = try func.allocReg(.int);
3614 defer func.register_manager.unlockReg(tmp_lock);
3615 _ = try func.addInst(.{
3616 .tag = .pseudo_load_tlv,
3617 .data = .{ .reloc = .{
3618 .register = tmp_reg,
3619 .atom_index = try func.owner.getSymbolIndex(func),
3620 .sym_index = tlv_sym_index,
3621 } },
3622 });
3623 try func.genCopy(ptr_ty, dest_mcv, .{ .register = tmp_reg });
3624 }
3625
3626 return func.finishAir(inst, dest_mcv, .{ .none, .none, .none });
3627}
3628
3629fn airTry(func: *Func, inst: Air.Inst.Index) !void {
3630 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3631 const extra = func.air.extraData(Air.Try, pl_op.payload);
3632 const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]);
3633 const operand_ty = func.typeOf(pl_op.operand);
3634 const result = try func.genTry(inst, pl_op.operand, body, operand_ty, false);
3635 return func.finishAir(inst, result, .{ .none, .none, .none });
3636}
3637
3638fn genTry(
3639 func: *Func,
3640 inst: Air.Inst.Index,
3641 operand: Air.Inst.Ref,
3642 body: []const Air.Inst.Index,
3643 operand_ty: Type,
3644 operand_is_ptr: bool,
3645) !MCValue {
3646 _ = operand_is_ptr;
3647
3648 const liveness_cond_br = func.liveness.getCondBr(inst);
3649
3650 const operand_mcv = try func.resolveInst(operand);
3651 const is_err_mcv = try func.isErr(null, operand_ty, operand_mcv);
3652
3653 // A branch to the false section. Uses beq. 1 is the default "true" state.
3654 const reloc = try func.condBr(Type.anyerror, is_err_mcv);
3655
3656 if (func.liveness.operandDies(inst, 0)) {
3657 if (operand.toIndex()) |operand_inst| try func.processDeath(operand_inst);
3658 }
3659
3660 func.scope_generation += 1;
3661 const state = try func.saveState();
3662
3663 for (liveness_cond_br.else_deaths) |death| try func.processDeath(death);
3664 try func.genBody(body);
3665 try func.restoreState(state, &.{}, .{
3666 .emit_instructions = false,
3667 .update_tracking = true,
3668 .resurrect = true,
3669 .close_scope = true,
3670 });
3671
3672 func.performReloc(reloc);
3673
3674 for (liveness_cond_br.then_deaths) |death| try func.processDeath(death);
3675
3676 const result = if (func.liveness.isUnused(inst))
3677 .unreach
3678 else
3679 try func.genUnwrapErrUnionPayloadMir(operand_ty, operand_mcv);
3680
3681 return result;
3682}
3683
3684fn airSlicePtr(func: *Func, inst: Air.Inst.Index) !void {
3685 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3686 const result = result: {
3687 const src_mcv = try func.resolveInst(ty_op.operand);
3688 if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
3689
3690 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
3691 const dst_ty = func.typeOfIndex(inst);
3692 try func.genCopy(dst_ty, dst_mcv, src_mcv);
3693 break :result dst_mcv;
3694 };
3695 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3696}
3697
3698fn airSliceLen(func: *Func, inst: Air.Inst.Index) !void {
3699 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3700 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3701 const src_mcv = try func.resolveInst(ty_op.operand);
3702 const ty = func.typeOfIndex(inst);
3703
3704 switch (src_mcv) {
3705 .load_frame => |frame_addr| {
3706 const len_mcv: MCValue = .{ .load_frame = .{
3707 .index = frame_addr.index,
3708 .off = frame_addr.off + 8,
3709 } };
3710 if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
3711
3712 const dst_mcv = try func.allocRegOrMem(ty, inst, true);
3713 try func.genCopy(Type.u64, dst_mcv, len_mcv);
3714 break :result dst_mcv;
3715 },
3716 .register_pair => |pair| {
3717 const len_mcv: MCValue = .{ .register = pair[1] };
3718
3719 if (func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv;
3720
3721 const dst_mcv = try func.allocRegOrMem(ty, inst, true);
3722 try func.genCopy(Type.u64, dst_mcv, len_mcv);
3723 break :result dst_mcv;
3724 },
3725 else => return func.fail("TODO airSliceLen for {}", .{src_mcv}),
3726 }
3727 };
3728 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3729}
3730
3731fn airPtrSliceLenPtr(func: *Func, inst: Air.Inst.Index) !void {
3732 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3733 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3734 const src_mcv = try func.resolveInst(ty_op.operand);
3735
3736 const dst_reg, const dst_lock = try func.allocReg(.int);
3737 defer func.register_manager.unlockReg(dst_lock);
3738 const dst_mcv: MCValue = .{ .register = dst_reg };
3739
3740 try func.genCopy(Type.u64, dst_mcv, src_mcv.offset(8));
3741 break :result dst_mcv;
3742 };
3743 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3744}
3745
3746fn airPtrSlicePtrPtr(func: *Func, inst: Air.Inst.Index) !void {
3747 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3748
3749 const opt_mcv = try func.resolveInst(ty_op.operand);
3750 const dst_mcv = if (func.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
3751 opt_mcv
3752 else
3753 try func.copyToNewRegister(inst, opt_mcv);
3754 return func.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
3755}
3756
3757fn airSliceElemVal(func: *Func, inst: Air.Inst.Index) !void {
3758 const pt = func.pt;
3759 const zcu = pt.zcu;
3760 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3761
3762 const result: MCValue = result: {
3763 const elem_ty = func.typeOfIndex(inst);
3764 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
3765
3766 const slice_ty = func.typeOf(bin_op.lhs);
3767 const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
3768 const elem_ptr = try func.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
3769 const dst_mcv = try func.allocRegOrMem(elem_ty, inst, false);
3770 try func.load(dst_mcv, elem_ptr, slice_ptr_field_type);
3771 break :result dst_mcv;
3772 };
3773 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3774}
3775
3776fn airSliceElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3777 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3778 const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3779 const dst_mcv = try func.genSliceElemPtr(extra.lhs, extra.rhs);
3780 return func.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
3781}
3782
3783fn genSliceElemPtr(func: *Func, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
3784 const pt = func.pt;
3785 const zcu = pt.zcu;
3786 const slice_ty = func.typeOf(lhs);
3787 const slice_mcv = try func.resolveInst(lhs);
3788 const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
3789 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3790 else => null,
3791 };
3792 defer if (slice_mcv_lock) |lock| func.register_manager.unlockReg(lock);
3793
3794 const elem_ty = slice_ty.childType(zcu);
3795 const elem_size = elem_ty.abiSize(zcu);
3796
3797 const index_ty = func.typeOf(rhs);
3798 const index_mcv = try func.resolveInst(rhs);
3799 const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
3800 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3801 else => null,
3802 };
3803 defer if (index_mcv_lock) |lock| func.register_manager.unlockReg(lock);
3804
3805 const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_size);
3806 const offset_reg_lock = func.register_manager.lockRegAssumeUnused(offset_reg);
3807 defer func.register_manager.unlockReg(offset_reg_lock);
3808
3809 const addr_reg, const addr_lock = try func.allocReg(.int);
3810 defer func.register_manager.unlockReg(addr_lock);
3811 try func.genSetReg(Type.u64, addr_reg, slice_mcv);
3812
3813 _ = try func.addInst(.{
3814 .tag = .add,
3815 .data = .{ .r_type = .{
3816 .rd = addr_reg,
3817 .rs1 = addr_reg,
3818 .rs2 = offset_reg,
3819 } },
3820 });
3821
3822 return .{ .register = addr_reg };
3823}
3824
3825fn airArrayElemVal(func: *Func, inst: Air.Inst.Index) !void {
3826 const pt = func.pt;
3827 const zcu = pt.zcu;
3828 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3829 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3830 const result_ty = func.typeOfIndex(inst);
3831
3832 const array_ty = func.typeOf(bin_op.lhs);
3833 const array_mcv = try func.resolveInst(bin_op.lhs);
3834
3835 const index_mcv = try func.resolveInst(bin_op.rhs);
3836 const index_ty = func.typeOf(bin_op.rhs);
3837
3838 const elem_ty = array_ty.childType(zcu);
3839 const elem_abi_size = elem_ty.abiSize(zcu);
3840
3841 const addr_reg, const addr_reg_lock = try func.allocReg(.int);
3842 defer func.register_manager.unlockReg(addr_reg_lock);
3843
3844 switch (array_mcv) {
3845 .register => {
3846 const frame_index = try func.allocFrameIndex(FrameAlloc.initType(array_ty, zcu));
3847 try func.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv);
3848 try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = .{ .index = frame_index } });
3849 },
3850 .load_frame => |frame_addr| {
3851 try func.genSetReg(Type.u64, addr_reg, .{ .lea_frame = frame_addr });
3852 },
3853 else => try func.genSetReg(Type.u64, addr_reg, array_mcv.address()),
3854 }
3855
3856 const dst_mcv = try func.allocRegOrMem(result_ty, inst, false);
3857
3858 if (array_ty.isVector(zcu)) {
3859 // we need to load the vector, vslidedown to get the element we want
3860 // and store that element in a load frame.
3861
3862 const src_reg, const src_lock = try func.allocReg(.vector);
3863 defer func.register_manager.unlockReg(src_lock);
3864
3865 // load the vector into a temporary register
3866 try func.genCopy(array_ty, .{ .register = src_reg }, .{ .indirect = .{ .reg = addr_reg } });
3867
3868 // we need to construct a 1xbitSize vector because of how lane splitting works in RISC-V
3869 const single_ty = try pt.vectorType(.{ .child = elem_ty.toIntern(), .len = 1 });
3870
3871 // we can do a shortcut here where we don't need a vslicedown
3872 // and can just copy to the frame index.
3873 if (!(index_mcv == .immediate and index_mcv.immediate == 0)) {
3874 const index_reg = try func.copyToTmpRegister(Type.u64, index_mcv);
3875
3876 _ = try func.addInst(.{
3877 .tag = .vslidedownvx,
3878 .data = .{ .r_type = .{
3879 .rd = src_reg,
3880 .rs1 = index_reg,
3881 .rs2 = src_reg,
3882 } },
3883 });
3884 }
3885
3886 try func.genCopy(single_ty, dst_mcv, .{ .register = src_reg });
3887 break :result dst_mcv;
3888 }
3889
3890 const offset_reg = try func.elemOffset(index_ty, index_mcv, elem_abi_size);
3891 const offset_lock = func.register_manager.lockRegAssumeUnused(offset_reg);
3892 defer func.register_manager.unlockReg(offset_lock);
3893 _ = try func.addInst(.{
3894 .tag = .add,
3895 .data = .{ .r_type = .{
3896 .rd = addr_reg,
3897 .rs1 = addr_reg,
3898 .rs2 = offset_reg,
3899 } },
3900 });
3901
3902 try func.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
3903 break :result dst_mcv;
3904 };
3905 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3906}
3907
3908fn airPtrElemVal(func: *Func, inst: Air.Inst.Index) !void {
3909 const is_volatile = false; // TODO
3910 const pt = func.pt;
3911 const zcu = pt.zcu;
3912 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3913 const base_ptr_ty = func.typeOf(bin_op.lhs);
3914
3915 const result: MCValue = if (!is_volatile and func.liveness.isUnused(inst)) .unreach else result: {
3916 const elem_ty = base_ptr_ty.elemType2(zcu);
3917 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
3918
3919 const base_ptr_mcv = try func.resolveInst(bin_op.lhs);
3920 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
3921 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3922 else => null,
3923 };
3924 defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3925
3926 const index_mcv = try func.resolveInst(bin_op.rhs);
3927 const index_lock: ?RegisterLock = switch (index_mcv) {
3928 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3929 else => null,
3930 };
3931 defer if (index_lock) |lock| func.register_manager.unlockReg(lock);
3932
3933 const elem_ptr_reg, const elem_ptr_lock = if (base_ptr_mcv.isRegister() and
3934 func.liveness.operandDies(inst, 0))
3935 .{ base_ptr_mcv.register, null }
3936 else blk: {
3937 const reg, const lock = try func.allocReg(.int);
3938 try func.genSetReg(base_ptr_ty, reg, base_ptr_mcv);
3939 break :blk .{ reg, lock };
3940 };
3941 defer if (elem_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3942
3943 try func.genBinOp(
3944 .ptr_add,
3945 base_ptr_mcv,
3946 base_ptr_ty,
3947 index_mcv,
3948 Type.u64,
3949 elem_ptr_reg,
3950 );
3951
3952 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
3953 const dst_lock = switch (dst_mcv) {
3954 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3955 else => null,
3956 };
3957 defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
3958
3959 try func.load(dst_mcv, .{ .register = elem_ptr_reg }, base_ptr_ty);
3960 break :result dst_mcv;
3961 };
3962 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
3963}
3964
3965fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3966 const pt = func.pt;
3967 const zcu = pt.zcu;
3968 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3969 const extra = func.air.extraData(Air.Bin, ty_pl.payload).data;
3970
3971 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
3972 const elem_ptr_ty = func.typeOfIndex(inst);
3973 const base_ptr_ty = func.typeOf(extra.lhs);
3974
3975 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
3976 @panic("audit");
3977 }
3978
3979 const base_ptr_mcv = try func.resolveInst(extra.lhs);
3980 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
3981 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3982 else => null,
3983 };
3984 defer if (base_ptr_lock) |lock| func.register_manager.unlockReg(lock);
3985
3986 const index_mcv = try func.resolveInst(extra.rhs);
3987 const index_lock: ?RegisterLock = switch (index_mcv) {
3988 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
3989 else => null,
3990 };
3991 defer if (index_lock) |lock| func.register_manager.unlockReg(lock);
3992
3993 const result_reg, const result_lock = try func.allocReg(.int);
3994 defer func.register_manager.unlockReg(result_lock);
3995
3996 try func.genBinOp(
3997 .ptr_add,
3998 base_ptr_mcv,
3999 base_ptr_ty,
4000 index_mcv,
4001 Type.u64,
4002 result_reg,
4003 );
4004
4005 break :result MCValue{ .register = result_reg };
4006 };
4007
4008 return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
4009}
4010
4011fn airSetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
4012 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4013 _ = bin_op;
4014 return func.fail("TODO implement airSetUnionTag for {}", .{func.target.cpu.arch});
4015 // return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4016}
4017
4018fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void {
4019 const pt = func.pt;
4020 const zcu = pt.zcu;
4021 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4022
4023 const tag_ty = func.typeOfIndex(inst);
4024 const union_ty = func.typeOf(ty_op.operand);
4025 const layout = union_ty.unionGetLayout(zcu);
4026
4027 if (layout.tag_size == 0) {
4028 return func.finishAir(inst, .none, .{ ty_op.operand, .none, .none });
4029 }
4030
4031 const operand = try func.resolveInst(ty_op.operand);
4032
4033 const frame_mcv = try func.allocRegOrMem(union_ty, null, false);
4034 try func.genCopy(union_ty, frame_mcv, operand);
4035
4036 const tag_abi_size = tag_ty.abiSize(zcu);
4037 const result_reg, const result_lock = try func.allocReg(.int);
4038 defer func.register_manager.unlockReg(result_lock);
4039
4040 switch (frame_mcv) {
4041 .load_frame => {
4042 if (tag_abi_size <= 8) {
4043 const off: i32 = if (layout.tag_align.compare(.lt, layout.payload_align))
4044 @intCast(layout.payload_size)
4045 else
4046 0;
4047
4048 try func.genCopy(
4049 tag_ty,
4050 .{ .register = result_reg },
4051 frame_mcv.offset(off),
4052 );
4053 } else {
4054 return func.fail(
4055 "TODO implement get_union_tag for ABI larger than 8 bytes and operand {}, tag {f}",
4056 .{ frame_mcv, tag_ty.fmt(pt) },
4057 );
4058 }
4059 },
4060 else => return func.fail("TODO: airGetUnionTag {s}", .{@tagName(operand)}),
4061 }
4062
4063 return func.finishAir(inst, .{ .register = result_reg }, .{ ty_op.operand, .none, .none });
4064}
4065
4066fn airClz(func: *Func, inst: Air.Inst.Index) !void {
4067 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4068 const operand = try func.resolveInst(ty_op.operand);
4069 const ty = func.typeOf(ty_op.operand);
4070
4071 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4072 const src_reg, const src_lock = try func.promoteReg(ty, operand);
4073 defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
4074
4075 const dst_reg: Register = if (func.reuseOperand(
4076 inst,
4077 ty_op.operand,
4078 0,
4079 operand,
4080 ) and operand == .register)
4081 operand.register
4082 else
4083 (try func.allocRegOrMem(func.typeOfIndex(inst), inst, true)).register;
4084
4085 const bit_size = ty.bitSize(func.pt.zcu);
4086 if (!math.isPowerOfTwo(bit_size)) try func.truncateRegister(ty, src_reg);
4087
4088 if (bit_size > 64) {
4089 return func.fail("TODO: airClz > 64 bits, found {d}", .{bit_size});
4090 }
4091
4092 _ = try func.addInst(.{
4093 .tag = switch (bit_size) {
4094 32 => .clzw,
4095 else => .clz,
4096 },
4097 .data = .{
4098 .r_type = .{
4099 .rs2 = .zero, // rs2 is 0 filled in the spec
4100 .rs1 = src_reg,
4101 .rd = dst_reg,
4102 },
4103 },
4104 });
4105
4106 if (!(bit_size == 32 or bit_size == 64)) {
4107 _ = try func.addInst(.{
4108 .tag = .addi,
4109 .data = .{ .i_type = .{
4110 .rd = dst_reg,
4111 .rs1 = dst_reg,
4112 .imm12 = Immediate.s(-@as(i12, @intCast(64 - bit_size % 64))),
4113 } },
4114 });
4115 }
4116
4117 break :result .{ .register = dst_reg };
4118 };
4119 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4120}
4121
4122fn airCtz(func: *Func, inst: Air.Inst.Index) !void {
4123 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4124 _ = ty_op;
4125 return func.fail("TODO: finish ctz", .{});
4126}
4127
4128fn airPopcount(func: *Func, inst: Air.Inst.Index) !void {
4129 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4130 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4131 const pt = func.pt;
4132 const zcu = pt.zcu;
4133
4134 const operand = try func.resolveInst(ty_op.operand);
4135 const src_ty = func.typeOf(ty_op.operand);
4136 const operand_reg, const operand_lock = try func.promoteReg(src_ty, operand);
4137 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
4138
4139 const dst_reg, const dst_lock = try func.allocReg(.int);
4140 defer func.register_manager.unlockReg(dst_lock);
4141
4142 const bit_size = src_ty.bitSize(zcu);
4143 switch (bit_size) {
4144 32, 64 => {},
4145 1...31, 33...63 => try func.truncateRegister(src_ty, operand_reg),
4146 else => return func.fail("TODO: airPopcount > 64 bits", .{}),
4147 }
4148
4149 _ = try func.addInst(.{
4150 .tag = if (bit_size <= 32) .cpopw else .cpop,
4151 .data = .{
4152 .r_type = .{
4153 .rd = dst_reg,
4154 .rs1 = operand_reg,
4155 .rs2 = @enumFromInt(0b00010), // this is the cpop funct5
4156 },
4157 },
4158 });
4159
4160 break :result .{ .register = dst_reg };
4161 };
4162 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4163}
4164
4165fn airAbs(func: *Func, inst: Air.Inst.Index) !void {
4166 const pt = func.pt;
4167 const zcu = pt.zcu;
4168 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4169 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4170 const ty = func.typeOf(ty_op.operand);
4171 const scalar_ty = ty.scalarType(zcu);
4172 const operand = try func.resolveInst(ty_op.operand);
4173
4174 switch (scalar_ty.zigTypeTag(zcu)) {
4175 .int => if (ty.zigTypeTag(zcu) == .vector) {
4176 return func.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)});
4177 } else {
4178 const int_info = scalar_ty.intInfo(zcu);
4179 const int_bits = int_info.bits;
4180 switch (int_bits) {
4181 32, 64 => {},
4182 else => return func.fail("TODO: airAbs Int size {d}", .{int_bits}),
4183 }
4184
4185 const return_mcv = try func.copyToNewRegister(inst, operand);
4186 const operand_reg = return_mcv.register;
4187
4188 const temp_reg, const temp_lock = try func.allocReg(.int);
4189 defer func.register_manager.unlockReg(temp_lock);
4190
4191 _ = try func.addInst(.{
4192 .tag = switch (int_bits) {
4193 32 => .sraiw,
4194 64 => .srai,
4195 else => unreachable,
4196 },
4197 .data = .{ .i_type = .{
4198 .rd = temp_reg,
4199 .rs1 = operand_reg,
4200 .imm12 = Immediate.u(int_bits - 1),
4201 } },
4202 });
4203
4204 _ = try func.addInst(.{
4205 .tag = .xor,
4206 .data = .{ .r_type = .{
4207 .rd = operand_reg,
4208 .rs1 = operand_reg,
4209 .rs2 = temp_reg,
4210 } },
4211 });
4212
4213 _ = try func.addInst(.{
4214 .tag = switch (int_bits) {
4215 32 => .subw,
4216 64 => .sub,
4217 else => unreachable,
4218 },
4219 .data = .{ .r_type = .{
4220 .rd = operand_reg,
4221 .rs1 = operand_reg,
4222 .rs2 = temp_reg,
4223 } },
4224 });
4225
4226 break :result return_mcv;
4227 },
4228 .float => {
4229 const float_bits = scalar_ty.floatBits(zcu.getTarget());
4230 const mnem: Mnemonic = switch (float_bits) {
4231 16 => return func.fail("TODO: airAbs 16-bit float", .{}),
4232 32 => .fsgnjxs,
4233 64 => .fsgnjxd,
4234 80 => return func.fail("TODO: airAbs 80-bit float", .{}),
4235 128 => return func.fail("TODO: airAbs 128-bit float", .{}),
4236 else => unreachable,
4237 };
4238
4239 const return_mcv = try func.copyToNewRegister(inst, operand);
4240 const operand_reg = return_mcv.register;
4241
4242 assert(operand_reg.class() == .float);
4243
4244 _ = try func.addInst(.{
4245 .tag = mnem,
4246 .data = .{
4247 .r_type = .{
4248 .rd = operand_reg,
4249 .rs1 = operand_reg,
4250 .rs2 = operand_reg,
4251 },
4252 },
4253 });
4254
4255 break :result return_mcv;
4256 },
4257 else => return func.fail("TODO: implement airAbs {f}", .{scalar_ty.fmt(pt)}),
4258 }
4259
4260 break :result .unreach;
4261 };
4262 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4263}
4264
4265fn airByteSwap(func: *Func, inst: Air.Inst.Index) !void {
4266 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4267 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4268 const pt = func.pt;
4269 const zcu = pt.zcu;
4270 const ty = func.typeOf(ty_op.operand);
4271 const operand = try func.resolveInst(ty_op.operand);
4272
4273 switch (ty.zigTypeTag(zcu)) {
4274 .int => {
4275 const int_bits = ty.intInfo(zcu).bits;
4276
4277 // bytes are no-op
4278 if (int_bits == 8 and func.reuseOperand(inst, ty_op.operand, 0, operand)) {
4279 return func.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
4280 }
4281
4282 const dest_mcv = try func.copyToNewRegister(inst, operand);
4283 const dest_reg = dest_mcv.register;
4284
4285 switch (int_bits) {
4286 16 => {
4287 const temp_reg, const temp_lock = try func.allocReg(.int);
4288 defer func.register_manager.unlockReg(temp_lock);
4289
4290 _ = try func.addInst(.{
4291 .tag = .srli,
4292 .data = .{ .i_type = .{
4293 .imm12 = Immediate.s(8),
4294 .rd = temp_reg,
4295 .rs1 = dest_reg,
4296 } },
4297 });
4298
4299 _ = try func.addInst(.{
4300 .tag = .slli,
4301 .data = .{ .i_type = .{
4302 .imm12 = Immediate.s(8),
4303 .rd = dest_reg,
4304 .rs1 = dest_reg,
4305 } },
4306 });
4307 _ = try func.addInst(.{
4308 .tag = .@"or",
4309 .data = .{ .r_type = .{
4310 .rd = dest_reg,
4311 .rs1 = dest_reg,
4312 .rs2 = temp_reg,
4313 } },
4314 });
4315 },
4316 else => return func.fail("TODO: {d} bits for airByteSwap", .{int_bits}),
4317 }
4318
4319 break :result dest_mcv;
4320 },
4321 else => return func.fail("TODO: airByteSwap {f}", .{ty.fmt(pt)}),
4322 }
4323 };
4324 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4325}
4326
4327fn airBitReverse(func: *Func, inst: Air.Inst.Index) !void {
4328 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4329 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airBitReverse for {}", .{func.target.cpu.arch});
4330 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4331}
4332
4333fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4334 const pt = func.pt;
4335 const zcu = pt.zcu;
4336 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4337 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4338 const ty = func.typeOf(un_op);
4339
4340 const operand = try func.resolveInst(un_op);
4341 const operand_bit_size = ty.bitSize(zcu);
4342
4343 if (!math.isPowerOfTwo(operand_bit_size))
4344 return func.fail("TODO: airUnaryMath non-pow 2", .{});
4345
4346 const operand_reg, const operand_lock = try func.promoteReg(ty, operand);
4347 defer if (operand_lock) |lock| func.register_manager.unlockReg(lock);
4348
4349 const dst_class = func.typeRegClass(ty);
4350 const dst_reg, const dst_lock = try func.allocReg(dst_class);
4351 defer func.register_manager.unlockReg(dst_lock);
4352
4353 switch (ty.zigTypeTag(zcu)) {
4354 .float => {
4355 assert(dst_class == .float);
4356
4357 switch (operand_bit_size) {
4358 16, 80, 128 => return func.fail("TODO: airUnaryMath Float bit-size {}", .{operand_bit_size}),
4359 32, 64 => {},
4360 else => unreachable,
4361 }
4362
4363 switch (tag) {
4364 .sqrt => {
4365 _ = try func.addInst(.{
4366 .tag = if (operand_bit_size == 64) .fsqrtd else .fsqrts,
4367 .data = .{
4368 .r_type = .{
4369 .rd = dst_reg,
4370 .rs1 = operand_reg,
4371 .rs2 = .f0, // unused, spec says it's 0
4372 },
4373 },
4374 });
4375 },
4376
4377 else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}),
4378 }
4379 },
4380 .int => {
4381 assert(dst_class == .int);
4382
4383 switch (tag) {
4384 else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}),
4385 }
4386 },
4387 else => return func.fail("TODO: airUnaryMath ty: {f}", .{ty.fmt(pt)}),
4388 }
4389
4390 break :result MCValue{ .register = dst_reg };
4391 };
4392
4393 return func.finishAir(inst, result, .{ un_op, .none, .none });
4394}
4395
4396fn reuseOperand(
4397 func: *Func,
4398 inst: Air.Inst.Index,
4399 operand: Air.Inst.Ref,
4400 op_index: Air.Liveness.OperandInt,
4401 mcv: MCValue,
4402) bool {
4403 return func.reuseOperandAdvanced(inst, operand, op_index, mcv, inst);
4404}
4405
4406fn reuseOperandAdvanced(
4407 func: *Func,
4408 inst: Air.Inst.Index,
4409 operand: Air.Inst.Ref,
4410 op_index: Air.Liveness.OperandInt,
4411 mcv: MCValue,
4412 maybe_tracked_inst: ?Air.Inst.Index,
4413) bool {
4414 if (!func.liveness.operandDies(inst, op_index))
4415 return false;
4416
4417 switch (mcv) {
4418 .register,
4419 .register_pair,
4420 => for (mcv.getRegs()) |reg| {
4421 // If it's in the registers table, need to associate the register(s) with the
4422 // new instruction.
4423 if (maybe_tracked_inst) |tracked_inst| {
4424 if (!func.register_manager.isRegFree(reg)) {
4425 if (RegisterManager.indexOfRegIntoTracked(reg)) |index| {
4426 func.register_manager.registers[index] = tracked_inst;
4427 }
4428 }
4429 } else func.register_manager.freeReg(reg);
4430 },
4431 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
4432 else => return false,
4433 }
4434
4435 // Prevent the operand deaths processing code from deallocating it.
4436 func.reused_operands.set(op_index);
4437 const op_inst = operand.toIndex().?;
4438 func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst);
4439
4440 return true;
4441}
4442
4443fn airLoad(func: *Func, inst: Air.Inst.Index) !void {
4444 const pt = func.pt;
4445 const zcu = pt.zcu;
4446 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4447 const elem_ty = func.typeOfIndex(inst);
4448
4449 const result: MCValue = result: {
4450 if (!elem_ty.hasRuntimeBits(zcu))
4451 break :result .none;
4452
4453 const ptr = try func.resolveInst(ty_op.operand);
4454 const is_volatile = func.typeOf(ty_op.operand).isVolatilePtr(zcu);
4455 if (func.liveness.isUnused(inst) and !is_volatile)
4456 break :result .unreach;
4457
4458 const elem_size = elem_ty.abiSize(zcu);
4459
4460 const dst_mcv: MCValue = blk: {
4461 // The MCValue that holds the pointer can be re-used as the value.
4462 // - "ptr" is 8 bytes, and if the element is more than that, we cannot reuse it.
4463 //
4464 // - "ptr" will be stored in an integer register, so the type that we're gonna
4465 // load into it must also be a type that can be inside of an integer register
4466 if (elem_size <= 8 and
4467 (if (ptr == .register) func.typeRegClass(elem_ty) == ptr.register.class() else true) and
4468 func.reuseOperand(inst, ty_op.operand, 0, ptr))
4469 {
4470 break :blk ptr;
4471 } else {
4472 break :blk try func.allocRegOrMem(elem_ty, inst, true);
4473 }
4474 };
4475
4476 try func.load(dst_mcv, ptr, func.typeOf(ty_op.operand));
4477 break :result dst_mcv;
4478 };
4479 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4480}
4481
4482fn load(func: *Func, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
4483 const pt = func.pt;
4484 const zcu = pt.zcu;
4485 const dst_ty = ptr_ty.childType(zcu);
4486
4487 log.debug("loading {}:{f} into {}", .{ ptr_mcv, ptr_ty.fmt(pt), dst_mcv });
4488
4489 switch (ptr_mcv) {
4490 .none,
4491 .undef,
4492 .unreach,
4493 .dead,
4494 .register_pair,
4495 .reserved_frame,
4496 => unreachable, // not a valid pointer
4497
4498 .immediate,
4499 .register,
4500 .register_offset,
4501 .lea_frame,
4502 .lea_symbol,
4503 => try func.genCopy(dst_ty, dst_mcv, ptr_mcv.deref()),
4504
4505 .memory,
4506 .indirect,
4507 .load_symbol,
4508 .load_frame,
4509 => {
4510 const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv);
4511 const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg);
4512 defer func.register_manager.unlockReg(addr_lock);
4513
4514 try func.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } });
4515 },
4516 .air_ref => |ptr_ref| try func.load(dst_mcv, try func.resolveInst(ptr_ref), ptr_ty),
4517 }
4518}
4519
4520fn airStore(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
4521 if (safety) {
4522 // TODO if the value is undef, write 0xaa bytes to dest
4523 } else {
4524 // TODO if the value is undef, don't lower this instruction
4525 }
4526 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4527 const ptr = try func.resolveInst(bin_op.lhs);
4528 const value = try func.resolveInst(bin_op.rhs);
4529 const ptr_ty = func.typeOf(bin_op.lhs);
4530
4531 try func.store(ptr, value, ptr_ty);
4532
4533 return func.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
4534}
4535
4536/// Loads `value` into the "payload" of `pointer`.
4537fn store(func: *Func, ptr_mcv: MCValue, src_mcv: MCValue, ptr_ty: Type) !void {
4538 const zcu = func.pt.zcu;
4539 const src_ty = ptr_ty.childType(zcu);
4540 log.debug("storing {}:{f} in {}:{f}", .{ src_mcv, src_ty.fmt(func.pt), ptr_mcv, ptr_ty.fmt(func.pt) });
4541
4542 switch (ptr_mcv) {
4543 .none => unreachable,
4544 .undef => unreachable,
4545 .unreach => unreachable,
4546 .dead => unreachable,
4547 .register_pair => unreachable,
4548 .reserved_frame => unreachable,
4549
4550 .immediate,
4551 .register,
4552 .register_offset,
4553 .lea_symbol,
4554 .lea_frame,
4555 => try func.genCopy(src_ty, ptr_mcv.deref(), src_mcv),
4556
4557 .memory,
4558 .indirect,
4559 .load_symbol,
4560 .load_frame,
4561 => {
4562 const addr_reg = try func.copyToTmpRegister(ptr_ty, ptr_mcv);
4563 const addr_lock = func.register_manager.lockRegAssumeUnused(addr_reg);
4564 defer func.register_manager.unlockReg(addr_lock);
4565
4566 try func.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
4567 },
4568 .air_ref => |ptr_ref| try func.store(try func.resolveInst(ptr_ref), src_mcv, ptr_ty),
4569 }
4570}
4571
4572fn airStructFieldPtr(func: *Func, inst: Air.Inst.Index) !void {
4573 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4574 const extra = func.air.extraData(Air.StructField, ty_pl.payload).data;
4575 const result = try func.structFieldPtr(inst, extra.struct_operand, extra.field_index);
4576 return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4577}
4578
4579fn airStructFieldPtrIndex(func: *Func, inst: Air.Inst.Index, index: u8) !void {
4580 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4581 const result = try func.structFieldPtr(inst, ty_op.operand, index);
4582 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4583}
4584
4585fn structFieldPtr(func: *Func, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue {
4586 const pt = func.pt;
4587 const zcu = pt.zcu;
4588 const ptr_field_ty = func.typeOfIndex(inst);
4589 const ptr_container_ty = func.typeOf(operand);
4590 const container_ty = ptr_container_ty.childType(zcu);
4591
4592 const field_offset: i32 = switch (container_ty.containerLayout(zcu)) {
4593 .auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)),
4594 .@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) +
4595 (if (zcu.typeToStruct(container_ty)) |struct_obj| zcu.structPackedFieldBitOffset(struct_obj, index) else 0) -
4596 ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8),
4597 };
4598
4599 const src_mcv = try func.resolveInst(operand);
4600 const dst_mcv = if (switch (src_mcv) {
4601 .immediate, .lea_frame => true,
4602 .register, .register_offset => func.reuseOperand(inst, operand, 0, src_mcv),
4603 else => false,
4604 }) src_mcv else try func.copyToNewRegister(inst, src_mcv);
4605 return dst_mcv.offset(field_offset);
4606}
4607
4608fn airStructFieldVal(func: *Func, inst: Air.Inst.Index) !void {
4609 const pt = func.pt;
4610 const zcu = pt.zcu;
4611
4612 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4613 const extra = func.air.extraData(Air.StructField, ty_pl.payload).data;
4614 const operand = extra.struct_operand;
4615 const index = extra.field_index;
4616
4617 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4618 const src_mcv = try func.resolveInst(operand);
4619 const struct_ty = func.typeOf(operand);
4620 const field_ty = struct_ty.fieldType(index, zcu);
4621 if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
4622
4623 const field_off: u32 = switch (struct_ty.containerLayout(zcu)) {
4624 .auto, .@"extern" => @intCast(struct_ty.structFieldOffset(index, zcu) * 8),
4625 .@"packed" => if (zcu.typeToStruct(struct_ty)) |struct_type|
4626 zcu.structPackedFieldBitOffset(struct_type, index)
4627 else
4628 0,
4629 };
4630
4631 switch (src_mcv) {
4632 .dead, .unreach => unreachable,
4633 .register => |src_reg| {
4634 const src_reg_lock = func.register_manager.lockRegAssumeUnused(src_reg);
4635 defer func.register_manager.unlockReg(src_reg_lock);
4636
4637 const dst_reg = if (field_off == 0)
4638 (try func.copyToNewRegister(inst, src_mcv)).register
4639 else
4640 try func.copyToTmpRegister(Type.u64, .{ .register = src_reg });
4641
4642 const dst_mcv: MCValue = .{ .register = dst_reg };
4643 const dst_lock = func.register_manager.lockReg(dst_reg);
4644 defer if (dst_lock) |lock| func.register_manager.unlockReg(lock);
4645
4646 if (field_off > 0) {
4647 _ = try func.addInst(.{
4648 .tag = .srli,
4649 .data = .{ .i_type = .{
4650 .imm12 = Immediate.u(@intCast(field_off)),
4651 .rd = dst_reg,
4652 .rs1 = dst_reg,
4653 } },
4654 });
4655 }
4656
4657 if (field_off == 0) {
4658 try func.truncateRegister(field_ty, dst_reg);
4659 }
4660
4661 break :result if (field_off == 0) dst_mcv else try func.copyToNewRegister(inst, dst_mcv);
4662 },
4663 .load_frame => {
4664 const field_abi_size: u32 = @intCast(field_ty.abiSize(zcu));
4665 if (field_off % 8 == 0) {
4666 const field_byte_off = @divExact(field_off, 8);
4667 const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref();
4668 const field_bit_size = field_ty.bitSize(zcu);
4669
4670 if (field_abi_size <= 8) {
4671 const int_ty = try pt.intType(
4672 if (field_ty.isAbiInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned,
4673 @intCast(field_bit_size),
4674 );
4675
4676 const dst_reg, const dst_lock = try func.allocReg(.int);
4677 const dst_mcv = MCValue{ .register = dst_reg };
4678 defer func.register_manager.unlockReg(dst_lock);
4679
4680 try func.genCopy(int_ty, dst_mcv, off_mcv);
4681 break :result try func.copyToNewRegister(inst, dst_mcv);
4682 }
4683
4684 const container_abi_size: u32 = @intCast(struct_ty.abiSize(zcu));
4685 const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and
4686 func.reuseOperand(inst, operand, 0, src_mcv))
4687 off_mcv
4688 else dst: {
4689 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4690 try func.genCopy(field_ty, dst_mcv, off_mcv);
4691 break :dst dst_mcv;
4692 };
4693 if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
4694 const tmp_reg, const tmp_lock = try func.allocReg(.int);
4695 defer func.register_manager.unlockReg(tmp_lock);
4696
4697 const hi_mcv =
4698 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
4699 try func.genSetReg(Type.u64, tmp_reg, hi_mcv);
4700 try func.genCopy(Type.u64, hi_mcv, .{ .register = tmp_reg });
4701 }
4702 break :result dst_mcv;
4703 }
4704
4705 return func.fail("TODO: airStructFieldVal load_frame field_off non multiple of 8", .{});
4706 },
4707 else => return func.fail("TODO: airStructField {s}", .{@tagName(src_mcv)}),
4708 }
4709 };
4710
4711 return func.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
4712}
4713
4714fn airFieldParentPtr(func: *Func, inst: Air.Inst.Index) !void {
4715 _ = inst;
4716 return func.fail("TODO implement codegen airFieldParentPtr", .{});
4717}
4718
4719fn genArgDbgInfo(func: *const Func, name: []const u8, ty: Type, mcv: MCValue) InnerError!void {
4720 assert(!func.mod.strip);
4721
4722 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
4723 // We aren't allowed to interact with linker state here.
4724 if (true) return;
4725 switch (func.debug_output) {
4726 .dwarf => |dw| switch (mcv) {
4727 .register => |reg| dw.genLocalDebugInfo(
4728 .local_arg,
4729 name,
4730 ty,
4731 .{ .reg = reg.dwarfNum() },
4732 ) catch |err| return func.fail("failed to generate debug info: {s}", .{@errorName(err)}),
4733 .load_frame => {},
4734 else => {},
4735 },
4736 .plan9 => {},
4737 .none => {},
4738 }
4739}
4740
4741fn airArg(func: *Func, inst: Air.Inst.Index) InnerError!void {
4742 const zcu = func.pt.zcu;
4743
4744 var arg_index = func.arg_index;
4745
4746 // we skip over args that have no bits
4747 while (func.args[arg_index] == .none) arg_index += 1;
4748 func.arg_index = arg_index + 1;
4749
4750 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
4751 const src_mcv = func.args[arg_index];
4752 const arg_ty = func.typeOfIndex(inst);
4753
4754 const dst_mcv = try func.allocRegOrMem(arg_ty, inst, false);
4755
4756 log.debug("airArg {} -> {}", .{ src_mcv, dst_mcv });
4757
4758 try func.genCopy(arg_ty, dst_mcv, src_mcv);
4759
4760 const arg = func.air.instructions.items(.data)[@intFromEnum(inst)].arg;
4761 // can delete `func.func_index` if this logic is moved to emit
4762 const func_zir = zcu.funcInfo(func.func_index).zir_body_inst.resolveFull(&zcu.intern_pool).?;
4763 const file = zcu.fileByIndex(func_zir.file);
4764 const zir = &file.zir.?;
4765 const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4766
4767 try func.genArgDbgInfo(name, arg_ty, src_mcv);
4768 break :result dst_mcv;
4769 };
4770
4771 return func.finishAir(inst, result, .{ .none, .none, .none });
4772}
4773
4774fn airTrap(func: *Func) !void {
4775 _ = try func.addInst(.{
4776 .tag = .unimp,
4777 .data = .none,
4778 });
4779 return func.finishAirBookkeeping();
4780}
4781
4782fn airBreakpoint(func: *Func) !void {
4783 _ = try func.addInst(.{
4784 .tag = .ebreak,
4785 .data = .none,
4786 });
4787 return func.finishAirBookkeeping();
4788}
4789
4790fn airRetAddr(func: *Func, inst: Air.Inst.Index) !void {
4791 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4792 try func.genCopy(Type.u64, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } });
4793 return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
4794}
4795
4796fn airFrameAddress(func: *Func, inst: Air.Inst.Index) !void {
4797 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
4798 try func.genCopy(Type.u64, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } });
4799 return func.finishAir(inst, dst_mcv, .{ .none, .none, .none });
4800}
4801
4802fn airCall(func: *Func, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
4803 if (modifier == .always_tail) return func.fail("TODO implement tail calls for riscv64", .{});
4804 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4805 const callee = pl_op.operand;
4806 const extra = func.air.extraData(Air.Call, pl_op.payload);
4807 const arg_refs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.args_len]);
4808
4809 const expected_num_args = 8;
4810 const ExpectedContents = extern struct {
4811 vals: [expected_num_args][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)),
4812 };
4813 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
4814 std.heap.stackFallback(@sizeOf(ExpectedContents), func.gpa);
4815 const allocator = stack.get();
4816
4817 const arg_tys = try allocator.alloc(Type, arg_refs.len);
4818 defer allocator.free(arg_tys);
4819 for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = func.typeOf(arg_ref);
4820
4821 const arg_vals = try allocator.alloc(MCValue, arg_refs.len);
4822 defer allocator.free(arg_vals);
4823 for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref };
4824
4825 const call_ret = try func.genCall(.{ .air = callee }, arg_tys, arg_vals);
4826
4827 var bt = func.liveness.iterateBigTomb(inst);
4828 try func.feed(&bt, pl_op.operand);
4829 for (arg_refs) |arg_ref| try func.feed(&bt, arg_ref);
4830
4831 const result = if (func.liveness.isUnused(inst)) .unreach else call_ret;
4832 return func.finishAirResult(inst, result);
4833}
4834
4835fn genCall(
4836 func: *Func,
4837 info: union(enum) {
4838 air: Air.Inst.Ref,
4839 lib: struct {
4840 return_type: InternPool.Index,
4841 param_types: []const InternPool.Index,
4842 lib: ?[]const u8 = null,
4843 callee: []const u8,
4844 },
4845 },
4846 arg_tys: []const Type,
4847 args: []const MCValue,
4848) !MCValue {
4849 const pt = func.pt;
4850 const zcu = pt.zcu;
4851
4852 const fn_ty = switch (info) {
4853 .air => |callee| fn_info: {
4854 const callee_ty = func.typeOf(callee);
4855 break :fn_info switch (callee_ty.zigTypeTag(zcu)) {
4856 .@"fn" => callee_ty,
4857 .pointer => callee_ty.childType(zcu),
4858 else => unreachable,
4859 };
4860 },
4861 .lib => |lib| try pt.funcType(.{
4862 .param_types = lib.param_types,
4863 .return_type = lib.return_type,
4864 .cc = func.target.cCallingConvention().?,
4865 }),
4866 };
4867
4868 const fn_info = zcu.typeToFunc(fn_ty).?;
4869
4870 const allocator = func.gpa;
4871
4872 const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len);
4873 defer allocator.free(var_args);
4874 for (var_args, arg_tys[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty;
4875
4876 var call_info = try func.resolveCallingConventionValues(fn_info, var_args);
4877 defer call_info.deinit(func);
4878
4879 // We need a properly aligned and sized call frame to be able to call this function.
4880 {
4881 const needed_call_frame = FrameAlloc.init(.{
4882 .size = call_info.stack_byte_count,
4883 .alignment = call_info.stack_align,
4884 });
4885 const frame_allocs_slice = func.frame_allocs.slice();
4886 const stack_frame_size =
4887 &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)];
4888 stack_frame_size.* = @max(stack_frame_size.*, needed_call_frame.abi_size);
4889 const stack_frame_align =
4890 &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)];
4891 stack_frame_align.* = stack_frame_align.max(needed_call_frame.abi_align);
4892 }
4893
4894 var reg_locks = std.array_list.Managed(?RegisterLock).init(allocator);
4895 defer reg_locks.deinit();
4896 try reg_locks.ensureTotalCapacity(8);
4897 defer for (reg_locks.items) |reg_lock| if (reg_lock) |lock| func.register_manager.unlockReg(lock);
4898
4899 const frame_indices = try allocator.alloc(FrameIndex, args.len);
4900 defer allocator.free(frame_indices);
4901
4902 switch (call_info.return_value.long) {
4903 .none, .unreach => {},
4904 .indirect => |reg_off| try func.register_manager.getReg(reg_off.reg, null),
4905 else => unreachable,
4906 }
4907 for (call_info.args, args, arg_tys, frame_indices) |dst_arg, src_arg, arg_ty, *frame_index| {
4908 switch (dst_arg) {
4909 .none => {},
4910 .register => |reg| {
4911 try func.register_manager.getReg(reg, null);
4912 try reg_locks.append(func.register_manager.lockReg(reg));
4913 },
4914 .register_pair => |regs| {
4915 for (regs) |reg| try func.register_manager.getReg(reg, null);
4916 try reg_locks.appendSlice(&func.register_manager.lockRegs(2, regs));
4917 },
4918 .indirect => |reg_off| {
4919 frame_index.* = try func.allocFrameIndex(FrameAlloc.initType(arg_ty, zcu));
4920 try func.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg);
4921 try func.register_manager.getReg(reg_off.reg, null);
4922 try reg_locks.append(func.register_manager.lockReg(reg_off.reg));
4923 },
4924 else => return func.fail("TODO: genCall set arg {s}", .{@tagName(dst_arg)}),
4925 }
4926 }
4927
4928 switch (call_info.return_value.long) {
4929 .none, .unreach => {},
4930 .indirect => |reg_off| {
4931 const ret_ty = Type.fromInterned(fn_info.return_type);
4932 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(ret_ty, zcu));
4933 try func.genSetReg(Type.u64, reg_off.reg, .{
4934 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
4935 });
4936 call_info.return_value.short = .{ .load_frame = .{ .index = frame_index } };
4937 try reg_locks.append(func.register_manager.lockReg(reg_off.reg));
4938 },
4939 else => unreachable,
4940 }
4941
4942 for (call_info.args, arg_tys, args, frame_indices) |dst_arg, arg_ty, src_arg, frame_index| {
4943 switch (dst_arg) {
4944 .none, .load_frame => {},
4945 .register_pair => try func.genCopy(arg_ty, dst_arg, src_arg),
4946 .register => |dst_reg| try func.genSetReg(
4947 arg_ty,
4948 dst_reg,
4949 src_arg,
4950 ),
4951 .indirect => |reg_off| try func.genSetReg(Type.u64, reg_off.reg, .{
4952 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
4953 }),
4954 else => return func.fail("TODO: genCall actual set {s}", .{@tagName(dst_arg)}),
4955 }
4956 }
4957
4958 // Due to incremental compilation, how function calls are generated depends
4959 // on linking.
4960 switch (info) {
4961 .air => |callee| {
4962 if (try func.air.value(callee, pt)) |func_value| {
4963 const func_key = zcu.intern_pool.indexToKey(func_value.ip_index);
4964 switch (switch (func_key) {
4965 else => func_key,
4966 .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
4967 .nav => |nav| zcu.intern_pool.indexToKey(zcu.navValue(nav).toIntern()),
4968 else => func_key,
4969 } else func_key,
4970 }) {
4971 .func => |func_val| {
4972 if (func.bin_file.cast(.elf)) |elf_file| {
4973 const zo = elf_file.zigObjectPtr().?;
4974 const sym_index = try zo.getOrCreateMetadataForNav(zcu, func_val.owner_nav);
4975
4976 if (func.mod.pic) {
4977 return func.fail("TODO: genCall pic", .{});
4978 } else {
4979 try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } });
4980 _ = try func.addInst(.{
4981 .tag = .jalr,
4982 .data = .{ .i_type = .{
4983 .rd = .ra,
4984 .rs1 = .ra,
4985 .imm12 = Immediate.s(0),
4986 } },
4987 });
4988 }
4989 } else unreachable; // not a valid riscv64 format
4990 },
4991 .@"extern" => |@"extern"| {
4992 const lib_name = @"extern".lib_name.toSlice(&zcu.intern_pool);
4993 const name = @"extern".name.toSlice(&zcu.intern_pool);
4994 const atom_index = try func.owner.getSymbolIndex(func);
4995
4996 const elf_file = func.bin_file.cast(.elf).?;
4997 _ = try func.addInst(.{
4998 .tag = .pseudo_extern_fn_reloc,
4999 .data = .{ .reloc = .{
5000 .register = .ra,
5001 .atom_index = atom_index,
5002 .sym_index = try elf_file.getGlobalSymbol(name, lib_name),
5003 } },
5004 });
5005 },
5006 else => return func.fail("TODO implement calling bitcasted functions", .{}),
5007 }
5008 } else {
5009 assert(func.typeOf(callee).zigTypeTag(zcu) == .pointer);
5010 const addr_reg, const addr_lock = try func.allocReg(.int);
5011 defer func.register_manager.unlockReg(addr_lock);
5012 try func.genSetReg(Type.u64, addr_reg, .{ .air_ref = callee });
5013
5014 _ = try func.addInst(.{
5015 .tag = .jalr,
5016 .data = .{ .i_type = .{
5017 .rd = .ra,
5018 .rs1 = addr_reg,
5019 .imm12 = Immediate.s(0),
5020 } },
5021 });
5022 }
5023 },
5024 .lib => return func.fail("TODO: lib func calls", .{}),
5025 }
5026
5027 // reset the vector settings as they might have changed in the function
5028 func.avl = null;
5029 func.vtype = null;
5030
5031 return call_info.return_value.short;
5032}
5033
5034fn airRet(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
5035 const pt = func.pt;
5036 const zcu = pt.zcu;
5037 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5038
5039 if (safety) {
5040 // safe
5041 } else {
5042 // not safe
5043 }
5044
5045 const ret_ty = func.fn_type.fnReturnType(zcu);
5046 switch (func.ret_mcv.short) {
5047 .none => {},
5048 .register,
5049 .register_pair,
5050 => {
5051 if (ret_ty.isVector(zcu)) {
5052 const bit_size = ret_ty.totalVectorBits(zcu);
5053
5054 // set the vtype to hold the entire vector's contents in a single element
5055 try func.setVl(.zero, 0, .{
5056 .vsew = switch (bit_size) {
5057 8 => .@"8",
5058 16 => .@"16",
5059 32 => .@"32",
5060 64 => .@"64",
5061 else => unreachable,
5062 },
5063 .vlmul = .m1,
5064 .vma = true,
5065 .vta = true,
5066 });
5067 }
5068
5069 try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op });
5070 },
5071 .indirect => |reg_off| {
5072 try func.register_manager.getReg(reg_off.reg, null);
5073 const lock = func.register_manager.lockRegAssumeUnused(reg_off.reg);
5074 defer func.register_manager.unlockReg(lock);
5075
5076 try func.genSetReg(Type.u64, reg_off.reg, func.ret_mcv.long);
5077 try func.genSetMem(
5078 .{ .reg = reg_off.reg },
5079 reg_off.off,
5080 ret_ty,
5081 .{ .air_ref = un_op },
5082 );
5083 },
5084 else => unreachable,
5085 }
5086
5087 func.ret_mcv.liveOut(func, inst);
5088 try func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5089
5090 // Just add space for an instruction, reloced this later
5091 const index = try func.addInst(.{
5092 .tag = .pseudo_j,
5093 .data = .{ .j_type = .{
5094 .rd = .zero,
5095 .inst = undefined,
5096 } },
5097 });
5098
5099 try func.exitlude_jump_relocs.append(func.gpa, index);
5100}
5101
5102fn airRetLoad(func: *Func, inst: Air.Inst.Index) !void {
5103 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5104 const ptr = try func.resolveInst(un_op);
5105
5106 const ptr_ty = func.typeOf(un_op);
5107 switch (func.ret_mcv.short) {
5108 .none => {},
5109 .register, .register_pair => try func.load(func.ret_mcv.short, ptr, ptr_ty),
5110 .indirect => |reg_off| try func.genSetReg(ptr_ty, reg_off.reg, ptr),
5111 else => unreachable,
5112 }
5113 func.ret_mcv.liveOut(func, inst);
5114 try func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5115
5116 // Just add space for an instruction, reloced this later
5117 const index = try func.addInst(.{
5118 .tag = .pseudo_j,
5119 .data = .{ .j_type = .{
5120 .rd = .zero,
5121 .inst = undefined,
5122 } },
5123 });
5124
5125 try func.exitlude_jump_relocs.append(func.gpa, index);
5126}
5127
5128fn airCmp(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
5129 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5130 const pt = func.pt;
5131 const zcu = pt.zcu;
5132 const ip = &zcu.intern_pool;
5133
5134 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5135 const lhs_ty = func.typeOf(bin_op.lhs);
5136
5137 switch (lhs_ty.zigTypeTag(zcu)) {
5138 .int,
5139 .@"enum",
5140 .bool,
5141 .pointer,
5142 .error_set,
5143 .optional,
5144 .@"struct",
5145 => {
5146 const int_ty = switch (lhs_ty.zigTypeTag(zcu)) {
5147 .@"enum" => lhs_ty.intTagType(zcu),
5148 .int => lhs_ty,
5149 .bool => Type.u1,
5150 .pointer => Type.u64,
5151 .error_set => Type.anyerror,
5152 .optional => blk: {
5153 const payload_ty = lhs_ty.optionalChild(zcu);
5154 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
5155 break :blk Type.u1;
5156 } else if (lhs_ty.isPtrLikeOptional(zcu)) {
5157 break :blk Type.u64;
5158 } else {
5159 return func.fail("TODO riscv cmp non-pointer optionals", .{});
5160 }
5161 },
5162 .@"struct" => blk: {
5163 const struct_obj = ip.loadStructType(lhs_ty.toIntern());
5164 assert(struct_obj.layout == .@"packed");
5165 const backing_index = struct_obj.backingIntTypeUnordered(ip);
5166 break :blk Type.fromInterned(backing_index);
5167 },
5168 else => unreachable,
5169 };
5170
5171 const int_info = int_ty.intInfo(zcu);
5172 if (int_info.bits <= 64) {
5173 break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
5174 } else {
5175 return func.fail("TODO riscv cmp for ints > 64 bits", .{});
5176 }
5177 },
5178 .float => {
5179 const float_bits = lhs_ty.floatBits(func.target);
5180 const float_reg_size: u32 = if (func.hasFeature(.d)) 64 else 32;
5181 if (float_bits > float_reg_size) {
5182 return func.fail("TODO: airCmp float > 64/32 bits", .{});
5183 }
5184 break :result try func.binOp(inst, tag, bin_op.lhs, bin_op.rhs);
5185 },
5186 else => unreachable,
5187 }
5188 };
5189
5190 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5191}
5192
5193fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void {
5194 _ = inst;
5195 return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch});
5196}
5197
5198fn airCmpLtErrorsLen(func: *Func, inst: Air.Inst.Index) !void {
5199 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5200 const operand = try func.resolveInst(un_op);
5201 _ = operand;
5202 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLtErrorsLen for {}", .{func.target.cpu.arch});
5203 return func.finishAir(inst, result, .{ un_op, .none, .none });
5204}
5205
5206fn airDbgStmt(func: *Func, inst: Air.Inst.Index) !void {
5207 const dbg_stmt = func.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
5208
5209 _ = try func.addInst(.{
5210 .tag = .pseudo_dbg_line_column,
5211 .data = .{ .pseudo_dbg_line_column = .{
5212 .line = dbg_stmt.line,
5213 .column = dbg_stmt.column,
5214 } },
5215 });
5216
5217 return func.finishAirBookkeeping();
5218}
5219
5220fn airDbgInlineBlock(func: *Func, inst: Air.Inst.Index) !void {
5221 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5222 const extra = func.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
5223 try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
5224}
5225
5226fn airDbgVar(func: *Func, inst: Air.Inst.Index) InnerError!void {
5227 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5228 const operand = pl_op.operand;
5229 const ty = func.typeOf(operand);
5230 const mcv = try func.resolveInst(operand);
5231 const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
5232
5233 const tag = func.air.instructions.items(.tag)[@intFromEnum(inst)];
5234 func.genVarDbgInfo(tag, ty, mcv, name.toSlice(func.air)) catch |err|
5235 return func.fail("failed to generate variable debug info: {s}", .{@errorName(err)});
5236
5237 return func.finishAir(inst, .unreach, .{ operand, .none, .none });
5238}
5239
5240fn genVarDbgInfo(
5241 func: Func,
5242 tag: Air.Inst.Tag,
5243 ty: Type,
5244 mcv: MCValue,
5245 name: []const u8,
5246) !void {
5247 // TODO: Add a pseudo-instruction or something to defer this work until Emit.
5248 // We aren't allowed to interact with linker state here.
5249 if (true) return;
5250 switch (func.debug_output) {
5251 .dwarf => |dwarf| {
5252 const loc: link.File.Dwarf.Loc = switch (mcv) {
5253 .register => |reg| .{ .reg = reg.dwarfNum() },
5254 .memory => |address| .{ .constu = address },
5255 .immediate => |x| .{ .constu = x },
5256 .none => .empty,
5257 else => blk: {
5258 // log.warn("TODO generate debug info for {}", .{mcv});
5259 break :blk .empty;
5260 },
5261 };
5262 try dwarf.genLocalDebugInfo(switch (tag) {
5263 else => unreachable,
5264 .dbg_var_ptr, .dbg_var_val => .local_var,
5265 .dbg_arg_inline => .local_arg,
5266 }, name, ty, loc);
5267 },
5268 .plan9 => {},
5269 .none => {},
5270 }
5271}
5272
5273fn airCondBr(func: *Func, inst: Air.Inst.Index) !void {
5274 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5275 const cond = try func.resolveInst(pl_op.operand);
5276 const cond_ty = func.typeOf(pl_op.operand);
5277 const extra = func.air.extraData(Air.CondBr, pl_op.payload);
5278 const then_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end..][0..extra.data.then_body_len]);
5279 const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5280 const liveness_cond_br = func.liveness.getCondBr(inst);
5281
5282 // If the condition dies here in this condbr instruction, process
5283 // that death now instead of later as this has an effect on
5284 // whether it needs to be spilled in the branches
5285 if (func.liveness.operandDies(inst, 0)) {
5286 if (pl_op.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5287 }
5288
5289 func.scope_generation += 1;
5290 const state = try func.saveState();
5291 const reloc = try func.condBr(cond_ty, cond);
5292
5293 for (liveness_cond_br.then_deaths) |death| try func.processDeath(death);
5294 try func.genBody(then_body);
5295 try func.restoreState(state, &.{}, .{
5296 .emit_instructions = false,
5297 .update_tracking = true,
5298 .resurrect = true,
5299 .close_scope = true,
5300 });
5301
5302 func.performReloc(reloc);
5303
5304 for (liveness_cond_br.else_deaths) |death| try func.processDeath(death);
5305 try func.genBody(else_body);
5306 try func.restoreState(state, &.{}, .{
5307 .emit_instructions = false,
5308 .update_tracking = true,
5309 .resurrect = true,
5310 .close_scope = true,
5311 });
5312
5313 // We already took care of pl_op.operand earlier, so there's nothing left to do.
5314 func.finishAirBookkeeping();
5315}
5316
5317fn condBr(func: *Func, cond_ty: Type, condition: MCValue) !Mir.Inst.Index {
5318 const cond_reg = try func.copyToTmpRegister(cond_ty, condition);
5319
5320 return try func.addInst(.{
5321 .tag = .beq,
5322 .data = .{
5323 .b_type = .{
5324 .rs1 = cond_reg,
5325 .rs2 = .zero,
5326 .inst = undefined,
5327 },
5328 },
5329 });
5330}
5331
5332fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
5333 const pt = func.pt;
5334 const zcu = pt.zcu;
5335 const pl_ty = opt_ty.optionalChild(zcu);
5336
5337 const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu))
5338 .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty }
5339 else
5340 .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool };
5341
5342 const return_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5343 assert(return_mcv == .register); // should not be larger 8 bytes
5344 const return_reg = return_mcv.register;
5345
5346 switch (opt_mcv) {
5347 .none,
5348 .unreach,
5349 .dead,
5350 .undef,
5351 .immediate,
5352 .register_offset,
5353 .lea_frame,
5354 .lea_symbol,
5355 .reserved_frame,
5356 .air_ref,
5357 .register_pair,
5358 => unreachable,
5359
5360 .register => |opt_reg| {
5361 if (some_info.off == 0) {
5362 _ = try func.addInst(.{
5363 .tag = .pseudo_compare,
5364 .data = .{
5365 .compare = .{
5366 .op = .eq,
5367 .rd = return_reg,
5368 .rs1 = opt_reg,
5369 .rs2 = try func.copyToTmpRegister(
5370 some_info.ty,
5371 .{ .immediate = 0 },
5372 ),
5373 .ty = Type.bool,
5374 },
5375 },
5376 });
5377 return return_mcv;
5378 }
5379 assert(some_info.ty.ip_index == .bool_type);
5380 const bit_offset: u7 = @intCast(some_info.off * 8);
5381
5382 try func.genBinOp(
5383 .shr,
5384 .{ .register = opt_reg },
5385 Type.u64,
5386 .{ .immediate = bit_offset },
5387 Type.u8,
5388 return_reg,
5389 );
5390 try func.truncateRegister(Type.u8, return_reg);
5391 try func.genBinOp(
5392 .cmp_eq,
5393 .{ .register = return_reg },
5394 Type.u64,
5395 .{ .immediate = 0 },
5396 Type.u8,
5397 return_reg,
5398 );
5399
5400 return return_mcv;
5401 },
5402
5403 .load_frame => {
5404 const opt_reg = try func.copyToTmpRegister(
5405 some_info.ty,
5406 opt_mcv.address().offset(some_info.off).deref(),
5407 );
5408 const opt_reg_lock = func.register_manager.lockRegAssumeUnused(opt_reg);
5409 defer func.register_manager.unlockReg(opt_reg_lock);
5410
5411 _ = try func.addInst(.{
5412 .tag = .pseudo_compare,
5413 .data = .{
5414 .compare = .{
5415 .op = .eq,
5416 .rd = return_reg,
5417 .rs1 = opt_reg,
5418 .rs2 = try func.copyToTmpRegister(
5419 some_info.ty,
5420 .{ .immediate = 0 },
5421 ),
5422 .ty = Type.bool,
5423 },
5424 },
5425 });
5426 return return_mcv;
5427 },
5428
5429 else => return func.fail("TODO: isNull {}", .{opt_mcv}),
5430 }
5431}
5432
5433fn airIsNull(func: *Func, inst: Air.Inst.Index) !void {
5434 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5435 const operand = try func.resolveInst(un_op);
5436 const ty = func.typeOf(un_op);
5437 const result = try func.isNull(inst, ty, operand);
5438 return func.finishAir(inst, result, .{ un_op, .none, .none });
5439}
5440
5441fn airIsNullPtr(func: *Func, inst: Air.Inst.Index) !void {
5442 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5443 const operand = try func.resolveInst(un_op);
5444 _ = operand;
5445 const ty = func.typeOf(un_op);
5446 _ = ty;
5447
5448 if (true) return func.fail("TODO: airIsNullPtr", .{});
5449
5450 return func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5451}
5452
5453fn airIsNonNull(func: *Func, inst: Air.Inst.Index) !void {
5454 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5455 const operand = try func.resolveInst(un_op);
5456 const ty = func.typeOf(un_op);
5457 const result = try func.isNull(inst, ty, operand);
5458 assert(result == .register);
5459
5460 _ = try func.addInst(.{
5461 .tag = .pseudo_not,
5462 .data = .{
5463 .rr = .{
5464 .rd = result.register,
5465 .rs = result.register,
5466 },
5467 },
5468 });
5469
5470 return func.finishAir(inst, result, .{ un_op, .none, .none });
5471}
5472
5473fn airIsNonNullPtr(func: *Func, inst: Air.Inst.Index) !void {
5474 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5475 const operand = try func.resolveInst(un_op);
5476 _ = operand;
5477 const ty = func.typeOf(un_op);
5478 _ = ty;
5479
5480 if (true) return func.fail("TODO: airIsNonNullPtr", .{});
5481
5482 return func.finishAir(inst, .unreach, .{ un_op, .none, .none });
5483}
5484
5485fn airIsErr(func: *Func, inst: Air.Inst.Index) !void {
5486 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5487 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5488 const operand = try func.resolveInst(un_op);
5489 const operand_ty = func.typeOf(un_op);
5490 break :result try func.isErr(inst, operand_ty, operand);
5491 };
5492 return func.finishAir(inst, result, .{ un_op, .none, .none });
5493}
5494
5495fn airIsErrPtr(func: *Func, inst: Air.Inst.Index) !void {
5496 const pt = func.pt;
5497 const zcu = pt.zcu;
5498 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5499 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5500 const operand_ptr = try func.resolveInst(un_op);
5501 const operand: MCValue = blk: {
5502 if (func.reuseOperand(inst, un_op, 0, operand_ptr)) {
5503 // The MCValue that holds the pointer can be re-used as the value.
5504 break :blk operand_ptr;
5505 } else {
5506 break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5507 }
5508 };
5509 try func.load(operand, operand_ptr, func.typeOf(un_op));
5510 const operand_ptr_ty = func.typeOf(un_op);
5511 const operand_ty = operand_ptr_ty.childType(zcu);
5512
5513 break :result try func.isErr(inst, operand_ty, operand);
5514 };
5515 return func.finishAir(inst, result, .{ un_op, .none, .none });
5516}
5517
5518/// Generates a compare instruction which will indicate if `eu_mcv` is an error.
5519///
5520/// Result is in the return register.
5521fn isErr(func: *Func, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
5522 _ = maybe_inst;
5523 const zcu = func.pt.zcu;
5524 const err_ty = eu_ty.errorUnionSet(zcu);
5525 if (err_ty.errorSetIsEmpty(zcu)) return MCValue{ .immediate = 0 }; // always false
5526 const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu));
5527
5528 const return_reg, const return_lock = try func.allocReg(.int);
5529 defer func.register_manager.unlockReg(return_lock);
5530
5531 switch (eu_mcv) {
5532 .register => |reg| {
5533 const eu_lock = func.register_manager.lockReg(reg);
5534 defer if (eu_lock) |lock| func.register_manager.unlockReg(lock);
5535
5536 try func.genCopy(eu_ty, .{ .register = return_reg }, eu_mcv);
5537
5538 if (err_off > 0) {
5539 try func.genBinOp(
5540 .shr,
5541 .{ .register = return_reg },
5542 eu_ty,
5543 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
5544 Type.u8,
5545 return_reg,
5546 );
5547 }
5548
5549 try func.genBinOp(
5550 .cmp_neq,
5551 .{ .register = return_reg },
5552 Type.anyerror,
5553 .{ .immediate = 0 },
5554 Type.u8,
5555 return_reg,
5556 );
5557 },
5558 .load_frame => |frame_addr| {
5559 try func.genBinOp(
5560 .cmp_neq,
5561 .{ .load_frame = .{
5562 .index = frame_addr.index,
5563 .off = frame_addr.off + err_off,
5564 } },
5565 Type.anyerror,
5566 .{ .immediate = 0 },
5567 Type.anyerror,
5568 return_reg,
5569 );
5570 },
5571 else => return func.fail("TODO implement isErr for {}", .{eu_mcv}),
5572 }
5573
5574 return .{ .register = return_reg };
5575}
5576
5577fn airIsNonErr(func: *Func, inst: Air.Inst.Index) !void {
5578 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5579 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5580 const operand = try func.resolveInst(un_op);
5581 const ty = func.typeOf(un_op);
5582 break :result try func.isNonErr(inst, ty, operand);
5583 };
5584 return func.finishAir(inst, result, .{ un_op, .none, .none });
5585}
5586
5587fn isNonErr(func: *Func, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
5588 const is_err_res = try func.isErr(inst, eu_ty, eu_mcv);
5589 switch (is_err_res) {
5590 .register => |reg| {
5591 _ = try func.addInst(.{
5592 .tag = .pseudo_not,
5593 .data = .{
5594 .rr = .{
5595 .rd = reg,
5596 .rs = reg,
5597 },
5598 },
5599 });
5600 return is_err_res;
5601 },
5602 // always false case
5603 .immediate => |imm| {
5604 assert(imm == 0);
5605 return MCValue{ .immediate = @intFromBool(imm == 0) };
5606 },
5607 else => unreachable,
5608 }
5609}
5610
5611fn airIsNonErrPtr(func: *Func, inst: Air.Inst.Index) !void {
5612 const pt = func.pt;
5613 const zcu = pt.zcu;
5614 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5615 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
5616 const operand_ptr = try func.resolveInst(un_op);
5617 const operand: MCValue = blk: {
5618 if (func.reuseOperand(inst, un_op, 0, operand_ptr)) {
5619 // The MCValue that holds the pointer can be re-used as the value.
5620 break :blk operand_ptr;
5621 } else {
5622 break :blk try func.allocRegOrMem(func.typeOfIndex(inst), inst, true);
5623 }
5624 };
5625 const operand_ptr_ty = func.typeOf(un_op);
5626 const operand_ty = operand_ptr_ty.childType(zcu);
5627
5628 try func.load(operand, operand_ptr, func.typeOf(un_op));
5629 break :result try func.isNonErr(inst, operand_ty, operand);
5630 };
5631 return func.finishAir(inst, result, .{ un_op, .none, .none });
5632}
5633
5634fn airLoop(func: *Func, inst: Air.Inst.Index) !void {
5635 // A loop is a setup to be able to jump back to the beginning.
5636 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5637 const loop = func.air.extraData(Air.Block, ty_pl.payload);
5638 const body: []const Air.Inst.Index = @ptrCast(func.air.extra.items[loop.end..][0..loop.data.body_len]);
5639
5640 func.scope_generation += 1;
5641 const state = try func.saveState();
5642
5643 try func.loops.putNoClobber(func.gpa, inst, .{
5644 .state = state,
5645 .jmp_target = @intCast(func.mir_instructions.len),
5646 });
5647 defer assert(func.loops.remove(inst));
5648
5649 try func.genBody(body);
5650
5651 func.finishAirBookkeeping();
5652}
5653
5654/// Send control flow to the `index` of `func.code`.
5655fn jump(func: *Func, index: Mir.Inst.Index) !Mir.Inst.Index {
5656 return func.addInst(.{
5657 .tag = .pseudo_j,
5658 .data = .{ .j_type = .{
5659 .rd = .zero,
5660 .inst = index,
5661 } },
5662 });
5663}
5664
5665fn airBlock(func: *Func, inst: Air.Inst.Index) !void {
5666 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5667 const extra = func.air.extraData(Air.Block, ty_pl.payload);
5668 try func.lowerBlock(inst, @ptrCast(func.air.extra.items[extra.end..][0..extra.data.body_len]));
5669}
5670
5671fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
5672 // A block is a setup to be able to jump to the end.
5673 const inst_tracking_i = func.inst_tracking.count();
5674 func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(.unreach));
5675
5676 func.scope_generation += 1;
5677 try func.blocks.putNoClobber(func.gpa, inst, .{ .state = func.initRetroactiveState() });
5678 const liveness = func.liveness.getBlock(inst);
5679
5680 // TODO emit debug info lexical block
5681 try func.genBody(body);
5682
5683 var block_data = func.blocks.fetchRemove(inst).?;
5684 defer block_data.value.deinit(func.gpa);
5685 if (block_data.value.relocs.items.len > 0) {
5686 try func.restoreState(block_data.value.state, liveness.deaths, .{
5687 .emit_instructions = false,
5688 .update_tracking = true,
5689 .resurrect = true,
5690 .close_scope = true,
5691 });
5692 for (block_data.value.relocs.items) |reloc| func.performReloc(reloc);
5693 }
5694
5695 if (std.debug.runtime_safety) assert(func.inst_tracking.getIndex(inst).? == inst_tracking_i);
5696 const tracking = &func.inst_tracking.values()[inst_tracking_i];
5697 if (func.liveness.isUnused(inst)) try tracking.die(func, inst);
5698 func.getValueIfFree(tracking.short, inst);
5699 func.finishAirBookkeeping();
5700}
5701
5702fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5703 const switch_br = func.air.unwrapSwitch(inst);
5704 const condition = try func.resolveInst(switch_br.operand);
5705
5706 // If the condition dies here in this switch instruction, process
5707 // that death now instead of later as this has an effect on
5708 // whether it needs to be spilled in the branches
5709 if (func.liveness.operandDies(inst, 0)) {
5710 if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5711 }
5712
5713 try func.lowerSwitchBr(inst, switch_br, condition);
5714
5715 // We already took care of pl_op.operand earlier, so there's nothing left to do
5716 func.finishAirBookkeeping();
5717}
5718
5719fn lowerSwitchBr(
5720 func: *Func,
5721 inst: Air.Inst.Index,
5722 switch_br: Air.UnwrappedSwitch,
5723 condition: MCValue,
5724) !void {
5725 const condition_ty = func.typeOf(switch_br.operand);
5726 const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
5727 defer func.gpa.free(liveness.deaths);
5728
5729 func.scope_generation += 1;
5730 const state = try func.saveState();
5731
5732 var it = switch_br.iterateCases();
5733 while (it.next()) |case| {
5734 var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
5735 defer func.gpa.free(relocs);
5736
5737 for (case.items, relocs[0..case.items.len]) |item, *reloc| {
5738 const item_mcv = try func.resolveInst(item);
5739
5740 const cond_lock = switch (condition) {
5741 .register => func.register_manager.lockRegAssumeUnused(condition.register),
5742 else => null,
5743 };
5744 defer if (cond_lock) |lock| func.register_manager.unlockReg(lock);
5745
5746 const cmp_reg, const cmp_lock = try func.allocReg(.int);
5747 defer func.register_manager.unlockReg(cmp_lock);
5748
5749 try func.genBinOp(
5750 .cmp_neq,
5751 condition,
5752 condition_ty,
5753 item_mcv,
5754 condition_ty,
5755 cmp_reg,
5756 );
5757
5758 reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg });
5759 }
5760
5761 for (case.ranges, relocs[case.items.len..]) |range, *reloc| {
5762 const min_mcv = try func.resolveInst(range[0]);
5763 const max_mcv = try func.resolveInst(range[1]);
5764 const cond_lock = switch (condition) {
5765 .register => func.register_manager.lockRegAssumeUnused(condition.register),
5766 else => null,
5767 };
5768 defer if (cond_lock) |lock| func.register_manager.unlockReg(lock);
5769
5770 const temp_cmp_reg, const temp_cmp_lock = try func.allocReg(.int);
5771 defer func.register_manager.unlockReg(temp_cmp_lock);
5772
5773 // is `condition` less than `min`? is "true", we've failed
5774 try func.genBinOp(
5775 .cmp_gte,
5776 condition,
5777 condition_ty,
5778 min_mcv,
5779 condition_ty,
5780 temp_cmp_reg,
5781 );
5782
5783 // if the compare was true, we will jump to the fail case and fall through
5784 // to the next checks
5785 const lt_fail_reloc = try func.condBr(condition_ty, .{ .register = temp_cmp_reg });
5786 try func.genBinOp(
5787 .cmp_gt,
5788 condition,
5789 condition_ty,
5790 max_mcv,
5791 condition_ty,
5792 temp_cmp_reg,
5793 );
5794
5795 reloc.* = try func.condBr(condition_ty, .{ .register = temp_cmp_reg });
5796 func.performReloc(lt_fail_reloc);
5797 }
5798
5799 const skip_case_reloc = try func.jump(undefined);
5800
5801 for (liveness.deaths[case.idx]) |operand| try func.processDeath(operand);
5802
5803 for (relocs) |reloc| func.performReloc(reloc);
5804 try func.genBody(case.body);
5805 try func.restoreState(state, &.{}, .{
5806 .emit_instructions = false,
5807 .update_tracking = true,
5808 .resurrect = true,
5809 .close_scope = true,
5810 });
5811
5812 func.performReloc(skip_case_reloc);
5813 }
5814
5815 if (switch_br.else_body_len > 0) {
5816 const else_body = it.elseBody();
5817
5818 const else_deaths = liveness.deaths.len - 1;
5819 for (liveness.deaths[else_deaths]) |operand| try func.processDeath(operand);
5820
5821 try func.genBody(else_body);
5822 try func.restoreState(state, &.{}, .{
5823 .emit_instructions = false,
5824 .update_tracking = true,
5825 .resurrect = true,
5826 .close_scope = true,
5827 });
5828 }
5829}
5830
5831fn airLoopSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5832 const switch_br = func.air.unwrapSwitch(inst);
5833 const condition = try func.resolveInst(switch_br.operand);
5834
5835 const mat_cond = if (condition.isMutable() and
5836 func.reuseOperand(inst, switch_br.operand, 0, condition))
5837 condition
5838 else mat_cond: {
5839 const ty = func.typeOf(switch_br.operand);
5840 const mat_cond = try func.allocRegOrMem(ty, inst, true);
5841 try func.genCopy(ty, mat_cond, condition);
5842 break :mat_cond mat_cond;
5843 };
5844 func.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(mat_cond));
5845
5846 // If the condition dies here in this switch instruction, process
5847 // that death now instead of later as this has an effect on
5848 // whether it needs to be spilled in the branches
5849 if (func.liveness.operandDies(inst, 0)) {
5850 if (switch_br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5851 }
5852
5853 func.scope_generation += 1;
5854 const state = try func.saveState();
5855
5856 try func.loops.putNoClobber(func.gpa, inst, .{
5857 .state = state,
5858 .jmp_target = @intCast(func.mir_instructions.len),
5859 });
5860 defer assert(func.loops.remove(inst));
5861
5862 // Stop tracking block result without forgetting tracking info
5863 try func.freeValue(mat_cond);
5864
5865 try func.lowerSwitchBr(inst, switch_br, mat_cond);
5866
5867 try func.processDeath(inst);
5868 func.finishAirBookkeeping();
5869}
5870
5871fn airSwitchDispatch(func: *Func, inst: Air.Inst.Index) !void {
5872 const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br;
5873
5874 const block_ty = func.typeOfIndex(br.block_inst);
5875 const block_tracking = func.inst_tracking.getPtr(br.block_inst).?;
5876 const loop_data = func.loops.getPtr(br.block_inst).?;
5877 done: {
5878 try func.getValue(block_tracking.short, null);
5879 const src_mcv = try func.resolveInst(br.operand);
5880
5881 if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
5882 try func.getValue(block_tracking.short, br.block_inst);
5883 // .long = .none to avoid merging operand and block result stack frames.
5884 const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv };
5885 try current_tracking.materializeUnsafe(func, br.block_inst, block_tracking.*);
5886 for (current_tracking.getRegs()) |src_reg| func.register_manager.freeReg(src_reg);
5887 break :done;
5888 }
5889
5890 try func.getValue(block_tracking.short, br.block_inst);
5891 const dst_mcv = block_tracking.short;
5892 try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand));
5893 break :done;
5894 }
5895
5896 // Process operand death so that it is properly accounted for in the State below.
5897 if (func.liveness.operandDies(inst, 0)) {
5898 if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5899 }
5900
5901 try func.restoreState(loop_data.state, &.{}, .{
5902 .emit_instructions = true,
5903 .update_tracking = false,
5904 .resurrect = false,
5905 .close_scope = false,
5906 });
5907
5908 // Emit a jump with a relocation. It will be patched up after the block ends.
5909 // Leave the jump offset undefined
5910 _ = try func.jump(loop_data.jmp_target);
5911
5912 // Stop tracking block result without forgetting tracking info
5913 try func.freeValue(block_tracking.short);
5914
5915 func.finishAirBookkeeping();
5916}
5917
5918fn performReloc(func: *Func, inst: Mir.Inst.Index) void {
5919 const tag = func.mir_instructions.items(.tag)[inst];
5920 const target: Mir.Inst.Index = @intCast(func.mir_instructions.len);
5921
5922 switch (tag) {
5923 .beq,
5924 .bne,
5925 => func.mir_instructions.items(.data)[inst].b_type.inst = target,
5926 .jal => func.mir_instructions.items(.data)[inst].j_type.inst = target,
5927 .pseudo_j => func.mir_instructions.items(.data)[inst].j_type.inst = target,
5928 else => std.debug.panic("TODO: performReloc {s}", .{@tagName(tag)}),
5929 }
5930}
5931
5932fn airBr(func: *Func, inst: Air.Inst.Index) !void {
5933 const zcu = func.pt.zcu;
5934 const br = func.air.instructions.items(.data)[@intFromEnum(inst)].br;
5935
5936 const block_ty = func.typeOfIndex(br.block_inst);
5937 const block_unused =
5938 !block_ty.hasRuntimeBitsIgnoreComptime(zcu) or func.liveness.isUnused(br.block_inst);
5939 const block_tracking = func.inst_tracking.getPtr(br.block_inst).?;
5940 const block_data = func.blocks.getPtr(br.block_inst).?;
5941 const first_br = block_data.relocs.items.len == 0;
5942 const block_result = result: {
5943 if (block_unused) break :result .none;
5944
5945 if (!first_br) try func.getValue(block_tracking.short, null);
5946 const src_mcv = try func.resolveInst(br.operand);
5947
5948 if (func.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
5949 if (first_br) break :result src_mcv;
5950
5951 try func.getValue(block_tracking.short, br.block_inst);
5952 try InstTracking.materializeUnsafe(
5953 // .long = .none to avoid merging operand and block result stack frames.
5954 .{ .long = .none, .short = src_mcv },
5955 func,
5956 br.block_inst,
5957 block_tracking.*,
5958 );
5959 try func.freeValue(src_mcv);
5960 break :result block_tracking.short;
5961 }
5962
5963 const dst_mcv = if (first_br) try func.allocRegOrMem(block_ty, br.block_inst, true) else dst: {
5964 try func.getValue(block_tracking.short, br.block_inst);
5965 break :dst block_tracking.short;
5966 };
5967 try func.genCopy(block_ty, dst_mcv, try func.resolveInst(br.operand));
5968 break :result dst_mcv;
5969 };
5970
5971 // Process operand death so that it is properly accounted for in the State below.
5972 if (func.liveness.operandDies(inst, 0)) {
5973 if (br.operand.toIndex()) |op_inst| try func.processDeath(op_inst);
5974 }
5975
5976 if (first_br) {
5977 block_tracking.* = InstTracking.init(block_result);
5978 try func.saveRetroactiveState(&block_data.state);
5979 } else try func.restoreState(block_data.state, &.{}, .{
5980 .emit_instructions = true,
5981 .update_tracking = false,
5982 .resurrect = false,
5983 .close_scope = false,
5984 });
5985
5986 // Emit a jump with a relocation. It will be patched up after the block ends.
5987 // Leave the jump offset undefined
5988 const jmp_reloc = try func.jump(undefined);
5989 try block_data.relocs.append(func.gpa, jmp_reloc);
5990
5991 // Stop tracking block result without forgetting tracking info
5992 try func.freeValue(block_tracking.short);
5993
5994 func.finishAirBookkeeping();
5995}
5996
5997fn airRepeat(func: *Func, inst: Air.Inst.Index) !void {
5998 const loop_inst = func.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
5999 const repeat_info = func.loops.get(loop_inst).?;
6000 try func.restoreState(repeat_info.state, &.{}, .{
6001 .emit_instructions = true,
6002 .update_tracking = false,
6003 .resurrect = false,
6004 .close_scope = true,
6005 });
6006 _ = try func.jump(repeat_info.jmp_target);
6007 func.finishAirBookkeeping();
6008}
6009
6010fn airBoolOp(func: *Func, inst: Air.Inst.Index) !void {
6011 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6012 const tag: Air.Inst.Tag = func.air.instructions.items(.tag)[@intFromEnum(inst)];
6013
6014 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
6015 const lhs = try func.resolveInst(bin_op.lhs);
6016 const rhs = try func.resolveInst(bin_op.rhs);
6017 const lhs_ty = Type.bool;
6018 const rhs_ty = Type.bool;
6019
6020 const lhs_reg, const lhs_lock = try func.promoteReg(lhs_ty, lhs);
6021 defer if (lhs_lock) |lock| func.register_manager.unlockReg(lock);
6022
6023 const rhs_reg, const rhs_lock = try func.promoteReg(rhs_ty, rhs);
6024 defer if (rhs_lock) |lock| func.register_manager.unlockReg(lock);
6025
6026 const result_reg, const result_lock = try func.allocReg(.int);
6027 defer func.register_manager.unlockReg(result_lock);
6028
6029 _ = try func.addInst(.{
6030 .tag = if (tag == .bool_or) .@"or" else .@"and",
6031 .data = .{ .r_type = .{
6032 .rd = result_reg,
6033 .rs1 = lhs_reg,
6034 .rs2 = rhs_reg,
6035 } },
6036 });
6037
6038 // safety truncate
6039 if (func.wantSafety()) {
6040 _ = try func.addInst(.{
6041 .tag = .andi,
6042 .data = .{ .i_type = .{
6043 .rd = result_reg,
6044 .rs1 = result_reg,
6045 .imm12 = Immediate.s(1),
6046 } },
6047 });
6048 }
6049
6050 break :result .{ .register = result_reg };
6051 };
6052 return func.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6053}
6054
6055fn airAsm(func: *Func, inst: Air.Inst.Index) !void {
6056 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6057 const extra = func.air.extraData(Air.Asm, ty_pl.payload);
6058 const outputs_len = extra.data.flags.outputs_len;
6059 var extra_i: usize = extra.end;
6060 const outputs: []const Air.Inst.Ref =
6061 @ptrCast(func.air.extra.items[extra_i..][0..outputs_len]);
6062 extra_i += outputs.len;
6063 const inputs: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[extra_i..][0..extra.data.inputs_len]);
6064 extra_i += inputs.len;
6065
6066 var result: MCValue = .none;
6067 var args = std.array_list.Managed(MCValue).init(func.gpa);
6068 try args.ensureTotalCapacity(outputs.len + inputs.len);
6069 defer {
6070 for (args.items) |arg| if (arg.getReg()) |reg| func.register_manager.unlockReg(.{
6071 .tracked_index = RegisterManager.indexOfRegIntoTracked(reg) orelse continue,
6072 });
6073 args.deinit();
6074 }
6075 var arg_map = std.StringHashMap(u8).init(func.gpa);
6076 try arg_map.ensureTotalCapacity(@intCast(outputs.len + inputs.len));
6077 defer arg_map.deinit();
6078
6079 var outputs_extra_i = extra_i;
6080 for (outputs) |output| {
6081 const extra_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6082 const constraint = mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[extra_i..]), 0);
6083 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6084 // This equation accounts for the fact that even if we have exactly 4 bytes
6085 // for the string, we still use the next u32 for the null terminator.
6086 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6087
6088 const is_read = switch (constraint[0]) {
6089 '=' => false,
6090 '+' => read: {
6091 if (output == .none) return func.fail(
6092 "read-write constraint unsupported for asm result: '{s}'",
6093 .{constraint},
6094 );
6095 break :read true;
6096 },
6097 else => return func.fail("invalid constraint: '{s}'", .{constraint}),
6098 };
6099 const is_early_clobber = constraint[1] == '&';
6100 const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..];
6101 const arg_mcv: MCValue = arg_mcv: {
6102 const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "m"))
6103 if (output != .none) null else return func.fail(
6104 "memory constraint unsupported for asm result: '{s}'",
6105 .{constraint},
6106 )
6107 else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}"))
6108 parseRegName(rest["{".len .. rest.len - "}".len]) orelse
6109 return func.fail("invalid register constraint: '{s}'", .{constraint})
6110 else if (rest.len == 1 and std.ascii.isDigit(rest[0])) {
6111 const index = std.fmt.charToDigit(rest[0], 10) catch unreachable;
6112 if (index >= args.items.len) return func.fail("constraint out of bounds: '{s}'", .{
6113 constraint,
6114 });
6115 break :arg_mcv args.items[index];
6116 } else return func.fail("invalid constraint: '{s}'", .{constraint});
6117 break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: {
6118 const ptr_mcv = try func.resolveInst(output);
6119 switch (ptr_mcv) {
6120 .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_|
6121 break :arg ptr_mcv.deref(),
6122 .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(),
6123 else => {},
6124 }
6125 break :arg .{ .indirect = .{ .reg = try func.copyToTmpRegister(Type.usize, ptr_mcv) } };
6126 };
6127 };
6128 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
6129 _ = func.register_manager.lockReg(reg);
6130 };
6131 if (!mem.eql(u8, name, "_"))
6132 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
6133 args.appendAssumeCapacity(arg_mcv);
6134 if (output == .none) result = arg_mcv;
6135 if (is_read) try func.load(arg_mcv, .{ .air_ref = output }, func.typeOf(output));
6136 }
6137
6138 for (inputs) |input| {
6139 const input_bytes = mem.sliceAsBytes(func.air.extra.items[extra_i..]);
6140 const constraint = mem.sliceTo(input_bytes, 0);
6141 const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0);
6142 // This equation accounts for the fact that even if we have exactly 4 bytes
6143 // for the string, we still use the next u32 for the null terminator.
6144 extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6145
6146 const ty = func.typeOf(input);
6147 const input_mcv = try func.resolveInst(input);
6148 const arg_mcv: MCValue = if (mem.eql(u8, constraint, "X"))
6149 input_mcv
6150 else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: {
6151 const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse
6152 return func.fail("invalid register constraint: '{s}'", .{constraint});
6153 try func.register_manager.getReg(reg, null);
6154 try func.genSetReg(ty, reg, input_mcv);
6155 break :arg .{ .register = reg };
6156 } else if (mem.eql(u8, constraint, "r")) arg: {
6157 switch (input_mcv) {
6158 .register => break :arg input_mcv,
6159 else => {},
6160 }
6161 const temp_reg = try func.copyToTmpRegister(ty, input_mcv);
6162 break :arg .{ .register = temp_reg };
6163 } else return func.fail("invalid input constraint: '{s}'", .{constraint});
6164 if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| {
6165 _ = func.register_manager.lockReg(reg);
6166 };
6167 if (!mem.eql(u8, name, "_"))
6168 arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len));
6169 args.appendAssumeCapacity(arg_mcv);
6170 }
6171
6172 const zcu = func.pt.zcu;
6173 const ip = &zcu.intern_pool;
6174 const aggregate = ip.indexToKey(extra.data.clobbers).aggregate;
6175 const struct_type: Type = .fromInterned(aggregate.ty);
6176 switch (aggregate.storage) {
6177 .elems => |elems| for (elems, 0..) |elem, i| {
6178 switch (elem) {
6179 .bool_true => {
6180 const clobber = struct_type.structFieldName(i, zcu).toSlice(ip).?;
6181 assert(clobber.len != 0);
6182 if (std.mem.eql(u8, clobber, "memory")) {
6183 // nothing really to do
6184 } else {
6185 try func.register_manager.getReg(parseRegName(clobber) orelse
6186 return func.fail("invalid clobber: '{s}'", .{clobber}), null);
6187 }
6188 },
6189 .bool_false => continue,
6190 else => unreachable,
6191 }
6192 },
6193 .repeated_elem => |elem| switch (elem) {
6194 .bool_true => @panic("TODO"),
6195 .bool_false => {},
6196 else => unreachable,
6197 },
6198 .bytes => @panic("TODO"),
6199 }
6200
6201 const Label = struct {
6202 target: Mir.Inst.Index = undefined,
6203 pending_relocs: std.ArrayList(Mir.Inst.Index) = .empty,
6204
6205 const Kind = enum { definition, reference };
6206
6207 fn isValid(kind: Kind, name: []const u8) bool {
6208 for (name, 0..) |c, i| switch (c) {
6209 else => return false,
6210 '$' => if (i == 0) return false,
6211 '.' => {},
6212 '0'...'9' => if (i == 0) switch (kind) {
6213 .definition => if (name.len != 1) return false,
6214 .reference => {
6215 if (name.len != 2) return false;
6216 switch (name[1]) {
6217 else => return false,
6218 'B', 'F', 'b', 'f' => {},
6219 }
6220 },
6221 },
6222 '@', 'A'...'Z', '_', 'a'...'z' => {},
6223 };
6224 return name.len > 0;
6225 }
6226 };
6227 var labels: std.StringHashMapUnmanaged(Label) = .empty;
6228 defer {
6229 var label_it = labels.valueIterator();
6230 while (label_it.next()) |label| label.pending_relocs.deinit(func.gpa);
6231 labels.deinit(func.gpa);
6232 }
6233
6234 const asm_source = std.mem.sliceAsBytes(func.air.extra.items[extra_i..])[0..extra.data.source_len];
6235 var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;");
6236 next_line: while (line_it.next()) |line| {
6237 var mnem_it = mem.tokenizeAny(u8, line, " \t");
6238 const mnem_str = while (mnem_it.next()) |mnem_str| {
6239 if (mem.startsWith(u8, mnem_str, "#")) continue :next_line;
6240 if (mem.startsWith(u8, mnem_str, "//")) continue :next_line;
6241 if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str;
6242 const label_name = mnem_str[0 .. mnem_str.len - ":".len];
6243 if (!Label.isValid(.definition, label_name))
6244 return func.fail("invalid label: '{s}'", .{label_name});
6245
6246 const label_gop = try labels.getOrPut(func.gpa, label_name);
6247 if (!label_gop.found_existing) label_gop.value_ptr.* = .{} else {
6248 const anon = std.ascii.isDigit(label_name[0]);
6249 if (!anon and label_gop.value_ptr.pending_relocs.items.len == 0)
6250 return func.fail("redefined label: '{s}'", .{label_name});
6251 for (label_gop.value_ptr.pending_relocs.items) |pending_reloc|
6252 func.performReloc(pending_reloc);
6253 if (anon)
6254 label_gop.value_ptr.pending_relocs.clearRetainingCapacity()
6255 else
6256 label_gop.value_ptr.pending_relocs.clearAndFree(func.gpa);
6257 }
6258 label_gop.value_ptr.target = @intCast(func.mir_instructions.len);
6259 } else continue;
6260
6261 const instruction: union(enum) { mnem: Mnemonic, pseudo: Pseudo } =
6262 if (std.meta.stringToEnum(Mnemonic, mnem_str)) |mnem|
6263 .{ .mnem = mnem }
6264 else if (std.meta.stringToEnum(Pseudo, mnem_str)) |pseudo|
6265 .{ .pseudo = pseudo }
6266 else
6267 return func.fail("invalid mnem str '{s}'", .{mnem_str});
6268
6269 const Operand = union(enum) {
6270 none,
6271 reg: Register,
6272 imm: Immediate,
6273 inst: Mir.Inst.Index,
6274 sym: SymbolOffset,
6275 };
6276
6277 var ops: [4]Operand = .{.none} ** 4;
6278 var last_op = false;
6279 var op_it = mem.splitAny(u8, mnem_it.rest(), ",(");
6280 next_op: for (&ops) |*op| {
6281 const op_str = while (!last_op) {
6282 const full_str = op_it.next() orelse break :next_op;
6283 const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse
6284 mem.indexOf(u8, full_str, "//")) |comment|
6285 code: {
6286 last_op = true;
6287 break :code full_str[0..comment];
6288 } else full_str;
6289 const trim_str = mem.trim(u8, code_str, " \t*");
6290 if (trim_str.len > 0) break trim_str;
6291 } else break;
6292
6293 if (parseRegName(op_str)) |reg| {
6294 op.* = .{ .reg = reg };
6295 } else if (std.fmt.parseInt(i12, op_str, 10)) |int| {
6296 op.* = .{ .imm = Immediate.s(int) };
6297 } else |_| if (mem.startsWith(u8, op_str, "%[")) {
6298 const mod_index = mem.indexOf(u8, op_str, "]@");
6299 const modifier = if (mod_index) |index|
6300 op_str[index + "]@".len ..]
6301 else
6302 "";
6303
6304 op.* = switch (args.items[
6305 arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse
6306 return func.fail("no matching constraint: '{s}'", .{op_str})
6307 ]) {
6308 .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: {
6309 assert(sym_off.off == 0);
6310 break :blk .{ .sym = sym_off };
6311 } else return func.fail("invalid modifier: '{s}'", .{modifier}),
6312 .register => |reg| if (modifier.len == 0)
6313 .{ .reg = reg }
6314 else
6315 return func.fail("invalid modified '{s}'", .{modifier}),
6316 else => return func.fail("invalid constraint: '{s}'", .{op_str}),
6317 };
6318 } else if (mem.endsWith(u8, op_str, ")")) {
6319 const reg = op_str[0 .. op_str.len - ")".len];
6320 const addr_reg = parseRegName(reg) orelse
6321 return func.fail("expected valid register, found '{s}'", .{reg});
6322
6323 op.* = .{ .reg = addr_reg };
6324 } else if (Label.isValid(.reference, op_str)) {
6325 const anon = std.ascii.isDigit(op_str[0]);
6326 const label_gop = try labels.getOrPut(func.gpa, op_str[0..if (anon) 1 else op_str.len]);
6327 if (!label_gop.found_existing) label_gop.value_ptr.* = .{};
6328 if (anon and (op_str[1] == 'b' or op_str[1] == 'B') and !label_gop.found_existing)
6329 return func.fail("undefined label: '{s}'", .{op_str});
6330 const pending_relocs = &label_gop.value_ptr.pending_relocs;
6331 if (if (anon)
6332 op_str[1] == 'f' or op_str[1] == 'F'
6333 else
6334 !label_gop.found_existing or pending_relocs.items.len > 0)
6335 try pending_relocs.append(func.gpa, @intCast(func.mir_instructions.len));
6336 op.* = .{ .inst = label_gop.value_ptr.target };
6337 } else return func.fail("invalid operand: '{s}'", .{op_str});
6338 } else if (op_it.next()) |op_str| return func.fail("extra operand: '{s}'", .{op_str});
6339
6340 switch (instruction) {
6341 .mnem => |mnem| {
6342 _ = (switch (ops[0]) {
6343 .none => try func.addInst(.{
6344 .tag = mnem,
6345 .data = .none,
6346 }),
6347 .reg => |reg1| switch (ops[1]) {
6348 .reg => |reg2| switch (ops[2]) {
6349 .imm => |imm1| try func.addInst(.{
6350 .tag = mnem,
6351 .data = .{ .i_type = .{
6352 .rd = reg1,
6353 .rs1 = reg2,
6354 .imm12 = imm1,
6355 } },
6356 }),
6357 else => error.InvalidInstruction,
6358 },
6359 .imm => |imm1| switch (ops[2]) {
6360 .reg => |reg2| switch (mnem) {
6361 .sd => try func.addInst(.{
6362 .tag = mnem,
6363 .data = .{ .i_type = .{
6364 .rd = reg2,
6365 .rs1 = reg1,
6366 .imm12 = imm1,
6367 } },
6368 }),
6369 .ld => try func.addInst(.{
6370 .tag = mnem,
6371 .data = .{ .i_type = .{
6372 .rd = reg1,
6373 .rs1 = reg2,
6374 .imm12 = imm1,
6375 } },
6376 }),
6377 else => error.InvalidInstruction,
6378 },
6379 else => error.InvalidInstruction,
6380 },
6381 .none => switch (mnem) {
6382 .jalr => try func.addInst(.{
6383 .tag = mnem,
6384 .data = .{ .i_type = .{
6385 .rd = .ra,
6386 .rs1 = reg1,
6387 .imm12 = Immediate.s(0),
6388 } },
6389 }),
6390 else => error.InvalidInstruction,
6391 },
6392 else => error.InvalidInstruction,
6393 },
6394 else => error.InvalidInstruction,
6395 }) catch |err| {
6396 switch (err) {
6397 error.InvalidInstruction => return func.fail(
6398 "invalid instruction: {s} {s} {s} {s} {s}",
6399 .{
6400 @tagName(mnem),
6401 @tagName(ops[0]),
6402 @tagName(ops[1]),
6403 @tagName(ops[2]),
6404 @tagName(ops[3]),
6405 },
6406 ),
6407 else => |e| return e,
6408 }
6409 };
6410 },
6411 .pseudo => |pseudo| {
6412 (@as(error{InvalidInstruction}!void, switch (pseudo) {
6413 .li => blk: {
6414 if (ops[0] != .reg or ops[1] != .imm) {
6415 break :blk error.InvalidInstruction;
6416 }
6417
6418 const reg = ops[0].reg;
6419 const imm = ops[1].imm;
6420
6421 try func.genSetReg(Type.usize, reg, .{ .immediate = imm.asBits(u64) });
6422 },
6423 .mv => blk: {
6424 if (ops[0] != .reg or ops[1] != .reg) {
6425 break :blk error.InvalidInstruction;
6426 }
6427
6428 const dst = ops[0].reg;
6429 const src = ops[1].reg;
6430
6431 if (dst.class() != .int or src.class() != .int) {
6432 return func.fail("pseudo instruction 'mv' only works on integer registers", .{});
6433 }
6434
6435 try func.genSetReg(Type.usize, dst, .{ .register = src });
6436 },
6437 .tail => blk: {
6438 if (ops[0] != .sym) {
6439 break :blk error.InvalidInstruction;
6440 }
6441
6442 const sym_offset = ops[0].sym;
6443 assert(sym_offset.off == 0);
6444
6445 const random_link_reg, const lock = try func.allocReg(.int);
6446 defer func.register_manager.unlockReg(lock);
6447
6448 _ = try func.addInst(.{
6449 .tag = .pseudo_extern_fn_reloc,
6450 .data = .{ .reloc = .{
6451 .register = random_link_reg,
6452 .atom_index = try func.owner.getSymbolIndex(func),
6453 .sym_index = sym_offset.sym,
6454 } },
6455 });
6456 },
6457 .ret => _ = try func.addInst(.{
6458 .tag = .jalr,
6459 .data = .{ .i_type = .{
6460 .rd = .zero,
6461 .rs1 = .ra,
6462 .imm12 = Immediate.s(0),
6463 } },
6464 }),
6465 .beqz => blk: {
6466 if (ops[0] != .reg or ops[1] != .inst) {
6467 break :blk error.InvalidInstruction;
6468 }
6469
6470 _ = try func.addInst(.{
6471 .tag = .beq,
6472 .data = .{ .b_type = .{
6473 .rs1 = ops[0].reg,
6474 .rs2 = .zero,
6475 .inst = ops[1].inst,
6476 } },
6477 });
6478 },
6479 })) catch |err| {
6480 switch (err) {
6481 error.InvalidInstruction => return func.fail(
6482 "invalid instruction: {s} {s} {s} {s} {s}",
6483 .{
6484 @tagName(pseudo),
6485 @tagName(ops[0]),
6486 @tagName(ops[1]),
6487 @tagName(ops[2]),
6488 @tagName(ops[3]),
6489 },
6490 ),
6491 else => |e| return e,
6492 }
6493 };
6494 },
6495 }
6496 }
6497
6498 var label_it = labels.iterator();
6499 while (label_it.next()) |label| if (label.value_ptr.pending_relocs.items.len > 0)
6500 return func.fail("undefined label: '{s}'", .{label.key_ptr.*});
6501
6502 for (outputs, args.items[0..outputs.len]) |output, arg_mcv| {
6503 const extra_bytes = mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]);
6504 const constraint =
6505 mem.sliceTo(mem.sliceAsBytes(func.air.extra.items[outputs_extra_i..]), 0);
6506 const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
6507 // This equation accounts for the fact that even if we have exactly 4 bytes
6508 // for the string, we still use the next u32 for the null terminator.
6509 outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4;
6510
6511 if (output == .none) continue;
6512 if (arg_mcv != .register) continue;
6513 if (constraint.len == 2 and std.ascii.isDigit(constraint[1])) continue;
6514 try func.store(.{ .air_ref = output }, arg_mcv, func.typeOf(output));
6515 }
6516
6517 simple: {
6518 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
6519 var buf_index: usize = 0;
6520 for (outputs) |output| {
6521 if (output == .none) continue;
6522
6523 if (buf_index >= buf.len) break :simple;
6524 buf[buf_index] = output;
6525 buf_index += 1;
6526 }
6527 if (buf_index + inputs.len > buf.len) break :simple;
6528 @memcpy(buf[buf_index..][0..inputs.len], inputs);
6529 return func.finishAir(inst, result, buf);
6530 }
6531 var bt = func.liveness.iterateBigTomb(inst);
6532 for (outputs) |output| if (output != .none) try func.feed(&bt, output);
6533 for (inputs) |input| try func.feed(&bt, input);
6534 return func.finishAirResult(inst, result);
6535}
6536
6537/// Sets the value of `dst_mcv` to the value of `src_mcv`.
6538fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
6539 // There isn't anything to store
6540 if (dst_mcv == .none) return;
6541
6542 if (!dst_mcv.isMutable()) {
6543 // panic so we can see the trace
6544 return std.debug.panic("tried to genCopy immutable: {s}", .{@tagName(dst_mcv)});
6545 }
6546
6547 const zcu = func.pt.zcu;
6548
6549 switch (dst_mcv) {
6550 .register => |reg| return func.genSetReg(ty, reg, src_mcv),
6551 .register_offset => |dst_reg_off| try func.genSetReg(ty, dst_reg_off.reg, switch (src_mcv) {
6552 .none,
6553 .unreach,
6554 .dead,
6555 .undef,
6556 => unreachable,
6557 .immediate,
6558 .register,
6559 .register_offset,
6560 => src_mcv.offset(-dst_reg_off.off),
6561 else => .{ .register_offset = .{
6562 .reg = try func.copyToTmpRegister(ty, src_mcv),
6563 .off = -dst_reg_off.off,
6564 } },
6565 }),
6566 .indirect => |reg_off| try func.genSetMem(
6567 .{ .reg = reg_off.reg },
6568 reg_off.off,
6569 ty,
6570 src_mcv,
6571 ),
6572 .load_frame => |frame_addr| try func.genSetMem(
6573 .{ .frame = frame_addr.index },
6574 frame_addr.off,
6575 ty,
6576 src_mcv,
6577 ),
6578 .load_symbol => {
6579 const addr_reg, const addr_lock = try func.allocReg(.int);
6580 defer func.register_manager.unlockReg(addr_lock);
6581
6582 try func.genSetReg(ty, addr_reg, dst_mcv.address());
6583 try func.genCopy(ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv);
6584 },
6585 .memory => return func.fail("TODO: genCopy memory", .{}),
6586 .register_pair => |dst_regs| {
6587 const src_info: ?struct { addr_reg: Register, addr_lock: ?RegisterLock } = switch (src_mcv) {
6588 .register_pair, .memory, .indirect, .load_frame => null,
6589 .load_symbol => src: {
6590 const src_addr_reg, const src_addr_lock = try func.promoteReg(Type.u64, src_mcv.address());
6591 errdefer func.register_manager.unlockReg(src_addr_lock);
6592
6593 break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock };
6594 },
6595 .air_ref => |src_ref| return func.genCopy(
6596 ty,
6597 dst_mcv,
6598 try func.resolveInst(src_ref),
6599 ),
6600 else => return func.fail("genCopy register_pair src: {}", .{src_mcv}),
6601 };
6602
6603 defer if (src_info) |info| {
6604 if (info.addr_lock) |lock| {
6605 func.register_manager.unlockReg(lock);
6606 }
6607 };
6608
6609 var part_disp: i32 = 0;
6610 for (dst_regs, try func.splitType(ty), 0..) |dst_reg, dst_ty, part_i| {
6611 try func.genSetReg(dst_ty, dst_reg, switch (src_mcv) {
6612 .register_pair => |src_regs| .{ .register = src_regs[part_i] },
6613 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
6614 .load_symbol => .{ .indirect = .{
6615 .reg = src_info.?.addr_reg,
6616 .off = part_disp,
6617 } },
6618 else => unreachable,
6619 });
6620 part_disp += @intCast(dst_ty.abiSize(zcu));
6621 }
6622 },
6623 else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }),
6624 }
6625}
6626
6627fn genInlineMemcpy(
6628 func: *Func,
6629 dst_ptr: MCValue,
6630 src_ptr: MCValue,
6631 len: MCValue,
6632) !void {
6633 const regs = try func.register_manager.allocRegs(4, .{null} ** 4, abi.Registers.Integer.temporary);
6634 const locks = func.register_manager.lockRegsAssumeUnused(4, regs);
6635 defer for (locks) |lock| func.register_manager.unlockReg(lock);
6636
6637 const count = regs[0];
6638 const tmp = regs[1];
6639 const src = regs[2];
6640 const dst = regs[3];
6641
6642 try func.genSetReg(Type.u64, count, len);
6643 try func.genSetReg(Type.u64, src, src_ptr);
6644 try func.genSetReg(Type.u64, dst, dst_ptr);
6645
6646 // if count is 0, there's nothing to copy
6647 _ = try func.addInst(.{
6648 .tag = .beq,
6649 .data = .{ .b_type = .{
6650 .rs1 = count,
6651 .rs2 = .zero,
6652 .inst = @intCast(func.mir_instructions.len + 9),
6653 } },
6654 });
6655
6656 // lb tmp, 0(src)
6657 const first_inst = try func.addInst(.{
6658 .tag = .lb,
6659 .data = .{
6660 .i_type = .{
6661 .rd = tmp,
6662 .rs1 = src,
6663 .imm12 = Immediate.s(0),
6664 },
6665 },
6666 });
6667
6668 // sb tmp, 0(dst)
6669 _ = try func.addInst(.{
6670 .tag = .sb,
6671 .data = .{
6672 .i_type = .{
6673 .rd = dst,
6674 .rs1 = tmp,
6675 .imm12 = Immediate.s(0),
6676 },
6677 },
6678 });
6679
6680 // dec count by 1
6681 _ = try func.addInst(.{
6682 .tag = .addi,
6683 .data = .{
6684 .i_type = .{
6685 .rd = count,
6686 .rs1 = count,
6687 .imm12 = Immediate.s(-1),
6688 },
6689 },
6690 });
6691
6692 // branch if count is 0
6693 _ = try func.addInst(.{
6694 .tag = .beq,
6695 .data = .{
6696 .b_type = .{
6697 .inst = @intCast(func.mir_instructions.len + 4), // points after the last inst
6698 .rs1 = count,
6699 .rs2 = .zero,
6700 },
6701 },
6702 });
6703
6704 // increment the pointers
6705 _ = try func.addInst(.{
6706 .tag = .addi,
6707 .data = .{
6708 .i_type = .{
6709 .rd = src,
6710 .rs1 = src,
6711 .imm12 = Immediate.s(1),
6712 },
6713 },
6714 });
6715
6716 _ = try func.addInst(.{
6717 .tag = .addi,
6718 .data = .{
6719 .i_type = .{
6720 .rd = dst,
6721 .rs1 = dst,
6722 .imm12 = Immediate.s(1),
6723 },
6724 },
6725 });
6726
6727 // jump back to start of loop
6728 _ = try func.addInst(.{
6729 .tag = .pseudo_j,
6730 .data = .{ .j_type = .{
6731 .rd = .zero,
6732 .inst = first_inst,
6733 } },
6734 });
6735}
6736
6737fn genInlineMemset(
6738 func: *Func,
6739 dst_ptr: MCValue,
6740 src_value: MCValue,
6741 len: MCValue,
6742) !void {
6743 const regs = try func.register_manager.allocRegs(3, .{null} ** 3, abi.Registers.Integer.temporary);
6744 const locks = func.register_manager.lockRegsAssumeUnused(3, regs);
6745 defer for (locks) |lock| func.register_manager.unlockReg(lock);
6746
6747 const count = regs[0];
6748 const src = regs[1];
6749 const dst = regs[2];
6750
6751 try func.genSetReg(Type.u64, count, len);
6752 try func.genSetReg(Type.u64, src, src_value);
6753 try func.genSetReg(Type.u64, dst, dst_ptr);
6754
6755 // sb src, 0(dst)
6756 const first_inst = try func.addInst(.{
6757 .tag = .sb,
6758 .data = .{
6759 .i_type = .{
6760 .rd = dst,
6761 .rs1 = src,
6762 .imm12 = Immediate.s(0),
6763 },
6764 },
6765 });
6766
6767 // dec count by 1
6768 _ = try func.addInst(.{
6769 .tag = .addi,
6770 .data = .{
6771 .i_type = .{
6772 .rd = count,
6773 .rs1 = count,
6774 .imm12 = Immediate.s(-1),
6775 },
6776 },
6777 });
6778
6779 // branch if count is 0
6780 _ = try func.addInst(.{
6781 .tag = .beq,
6782 .data = .{
6783 .b_type = .{
6784 .inst = @intCast(func.mir_instructions.len + 3), // points after the last inst
6785 .rs1 = count,
6786 .rs2 = .zero,
6787 },
6788 },
6789 });
6790
6791 // increment the pointers
6792 _ = try func.addInst(.{
6793 .tag = .addi,
6794 .data = .{
6795 .i_type = .{
6796 .rd = dst,
6797 .rs1 = dst,
6798 .imm12 = Immediate.s(1),
6799 },
6800 },
6801 });
6802
6803 // jump back to start of loop
6804 _ = try func.addInst(.{
6805 .tag = .pseudo_j,
6806 .data = .{ .j_type = .{
6807 .rd = .zero,
6808 .inst = first_inst,
6809 } },
6810 });
6811}
6812
6813/// Sets the value of `src_mcv` into `reg`. Assumes you have a lock on it.
6814fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError!void {
6815 const pt = func.pt;
6816 const zcu = pt.zcu;
6817 const abi_size: u32 = @intCast(ty.abiSize(zcu));
6818
6819 const max_size: u32 = switch (reg.class()) {
6820 .int => 64,
6821 .float => if (func.hasFeature(.d)) 64 else 32,
6822 .vector => 64, // TODO: calculate it from avl * vsew
6823 };
6824 if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size});
6825 const dst_reg_class = reg.class();
6826
6827 switch (src_mcv) {
6828 .unreach,
6829 .none,
6830 .dead,
6831 => unreachable,
6832 .undef => |sym_index| {
6833 if (!func.wantSafety())
6834 return;
6835
6836 if (sym_index) |index| {
6837 return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } });
6838 }
6839
6840 switch (abi_size) {
6841 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }),
6842 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }),
6843 3...4 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAA }),
6844 5...8 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAAAAAAAAAAAAAA }),
6845 else => unreachable,
6846 }
6847 },
6848 .immediate => |unsigned_x| {
6849 assert(dst_reg_class == .int);
6850
6851 const x: i64 = @bitCast(unsigned_x);
6852 if (math.minInt(i12) <= x and x <= math.maxInt(i12)) {
6853 _ = try func.addInst(.{
6854 .tag = .addi,
6855 .data = .{ .i_type = .{
6856 .rd = reg,
6857 .rs1 = .zero,
6858 .imm12 = Immediate.s(@intCast(x)),
6859 } },
6860 });
6861 } else if (math.minInt(i32) <= x and x <= math.maxInt(i32)) {
6862 const lo12: i12 = @truncate(x);
6863 const carry: i32 = if (lo12 < 0) 1 else 0;
6864 const hi20: i20 = @truncate((x >> 12) +% carry);
6865
6866 _ = try func.addInst(.{
6867 .tag = .lui,
6868 .data = .{ .u_type = .{
6869 .rd = reg,
6870 .imm20 = Immediate.s(hi20),
6871 } },
6872 });
6873 _ = try func.addInst(.{
6874 .tag = .addi,
6875 .data = .{ .i_type = .{
6876 .rd = reg,
6877 .rs1 = reg,
6878 .imm12 = Immediate.s(lo12),
6879 } },
6880 });
6881 } else {
6882 // TODO: use a more advanced myriad seq to do this without a reg.
6883 // see: https://github.com/llvm/llvm-project/blob/081a66ffacfe85a37ff775addafcf3371e967328/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp#L224
6884
6885 const temp, const temp_lock = try func.allocReg(.int);
6886 defer func.register_manager.unlockReg(temp_lock);
6887
6888 const lo32: i32 = @truncate(x);
6889 const carry: i32 = if (lo32 < 0) 1 else 0;
6890 const hi32: i32 = @truncate((x >> 32) +% carry);
6891
6892 try func.genSetReg(Type.i32, temp, .{ .immediate = @bitCast(@as(i64, lo32)) });
6893 try func.genSetReg(Type.i32, reg, .{ .immediate = @bitCast(@as(i64, hi32)) });
6894
6895 _ = try func.addInst(.{
6896 .tag = .slli,
6897 .data = .{ .i_type = .{
6898 .rd = reg,
6899 .rs1 = reg,
6900 .imm12 = Immediate.u(32),
6901 } },
6902 });
6903
6904 _ = try func.addInst(.{
6905 .tag = .add,
6906 .data = .{ .r_type = .{
6907 .rd = reg,
6908 .rs1 = reg,
6909 .rs2 = temp,
6910 } },
6911 });
6912 }
6913 },
6914 .register => |src_reg| {
6915 // If the registers are the same, nothing to do.
6916 if (src_reg.id() == reg.id())
6917 return;
6918
6919 // there is no instruction for loading the contents of a vector register
6920 // into an integer register, however we can cheat a bit by setting the element
6921 // size to the total size of the vector, and vmv.x.s will work then
6922 if (src_reg.class() == .vector) {
6923 try func.setVl(.zero, 0, .{
6924 .vsew = switch (ty.totalVectorBits(zcu)) {
6925 8 => .@"8",
6926 16 => .@"16",
6927 32 => .@"32",
6928 64 => .@"64",
6929 else => |vec_bits| return func.fail("TODO: genSetReg vec -> {s} bits {d}", .{
6930 @tagName(reg.class()),
6931 vec_bits,
6932 }),
6933 },
6934 .vlmul = .m1,
6935 .vta = true,
6936 .vma = true,
6937 });
6938 }
6939
6940 // mv reg, src_reg
6941 _ = try func.addInst(.{
6942 .tag = .pseudo_mv,
6943 .data = .{ .rr = .{
6944 .rd = reg,
6945 .rs = src_reg,
6946 } },
6947 });
6948 },
6949 // useful in cases like slice_ptr, which can easily reuse the operand
6950 // but we need to get only the pointer out.
6951 .register_pair => |pair| try func.genSetReg(ty, reg, .{ .register = pair[0] }),
6952 .load_frame => |frame| {
6953 if (reg.class() == .vector) {
6954 // vectors don't support an offset memory load so we need to put the true
6955 // address into a register before loading from it.
6956 const addr_reg, const addr_lock = try func.allocReg(.int);
6957 defer func.register_manager.unlockReg(addr_lock);
6958
6959 try func.genCopy(ty, .{ .register = addr_reg }, src_mcv.address());
6960 try func.genCopy(ty, .{ .register = reg }, .{ .indirect = .{ .reg = addr_reg } });
6961 } else {
6962 _ = try func.addInst(.{
6963 .tag = .pseudo_load_rm,
6964 .data = .{ .rm = .{
6965 .r = reg,
6966 .m = .{
6967 .base = .{ .frame = frame.index },
6968 .mod = .{
6969 .size = func.memSize(ty),
6970 .unsigned = ty.isUnsignedInt(zcu),
6971 .disp = frame.off,
6972 },
6973 },
6974 } },
6975 });
6976 }
6977 },
6978 .memory => |addr| {
6979 try func.genSetReg(ty, reg, .{ .immediate = addr });
6980
6981 _ = try func.addInst(.{
6982 .tag = .ld,
6983 .data = .{ .i_type = .{
6984 .rd = reg,
6985 .rs1 = reg,
6986 .imm12 = Immediate.u(0),
6987 } },
6988 });
6989 },
6990 .lea_frame, .register_offset => {
6991 _ = try func.addInst(.{
6992 .tag = .pseudo_lea_rm,
6993 .data = .{
6994 .rm = .{
6995 .r = reg,
6996 .m = switch (src_mcv) {
6997 .register_offset => |reg_off| .{
6998 .base = .{ .reg = reg_off.reg },
6999 .mod = .{
7000 .size = .byte, // the size doesn't matter
7001 .disp = reg_off.off,
7002 .unsigned = false,
7003 },
7004 },
7005 .lea_frame => |frame| .{
7006 .base = .{ .frame = frame.index },
7007 .mod = .{
7008 .size = .byte, // the size doesn't matter
7009 .disp = frame.off,
7010 .unsigned = false,
7011 },
7012 },
7013 else => unreachable,
7014 },
7015 },
7016 },
7017 });
7018 },
7019 .indirect => |reg_off| {
7020 const load_tag: Mnemonic = switch (reg.class()) {
7021 .float => switch (abi_size) {
7022 1 => unreachable, // Zig does not support 8-bit floats
7023 2 => return func.fail("TODO: genSetReg indirect 16-bit float", .{}),
7024 4 => .flw,
7025 8 => .fld,
7026 else => return std.debug.panic("TODO: genSetReg for float size {d}", .{abi_size}),
7027 },
7028 .int => switch (abi_size) {
7029 1...1 => .lb,
7030 2...2 => .lh,
7031 3...4 => .lw,
7032 5...8 => .ld,
7033 else => return std.debug.panic("TODO: genSetReg for int size {d}", .{abi_size}),
7034 },
7035 .vector => {
7036 assert(reg_off.off == 0);
7037
7038 // There is no vector instruction for loading with an offset to a base register,
7039 // so we need to get an offset register containing the address of the vector first
7040 // and load from it.
7041 const len = ty.vectorLen(zcu);
7042 const elem_ty = ty.childType(zcu);
7043 const elem_size = elem_ty.abiSize(zcu);
7044
7045 try func.setVl(.zero, len, .{
7046 .vsew = switch (elem_size) {
7047 1 => .@"8",
7048 2 => .@"16",
7049 4 => .@"32",
7050 8 => .@"64",
7051 else => unreachable,
7052 },
7053 .vlmul = .m1,
7054 .vma = true,
7055 .vta = true,
7056 });
7057
7058 _ = try func.addInst(.{
7059 .tag = .pseudo_load_rm,
7060 .data = .{ .rm = .{
7061 .r = reg,
7062 .m = .{
7063 .base = .{ .reg = reg_off.reg },
7064 .mod = .{
7065 .size = func.memSize(elem_ty),
7066 .unsigned = false,
7067 .disp = 0,
7068 },
7069 },
7070 } },
7071 });
7072
7073 return;
7074 },
7075 };
7076
7077 _ = try func.addInst(.{
7078 .tag = load_tag,
7079 .data = .{ .i_type = .{
7080 .rd = reg,
7081 .rs1 = reg_off.reg,
7082 .imm12 = Immediate.s(reg_off.off),
7083 } },
7084 });
7085 },
7086 .lea_symbol => |sym_off| {
7087 assert(sym_off.off == 0);
7088 const atom_index = try func.owner.getSymbolIndex(func);
7089
7090 _ = try func.addInst(.{
7091 .tag = .pseudo_load_symbol,
7092 .data = .{ .reloc = .{
7093 .register = reg,
7094 .atom_index = atom_index,
7095 .sym_index = sym_off.sym,
7096 } },
7097 });
7098 },
7099 .load_symbol => {
7100 const addr_reg, const addr_lock = try func.allocReg(.int);
7101 defer func.register_manager.unlockReg(addr_lock);
7102
7103 try func.genSetReg(ty, addr_reg, src_mcv.address());
7104 try func.genSetReg(ty, reg, .{ .indirect = .{ .reg = addr_reg } });
7105 },
7106 .air_ref => |ref| try func.genSetReg(ty, reg, try func.resolveInst(ref)),
7107 else => return func.fail("TODO: genSetReg {s}", .{@tagName(src_mcv)}),
7108 }
7109}
7110
7111fn genSetMem(
7112 func: *Func,
7113 base: Memory.Base,
7114 disp: i32,
7115 ty: Type,
7116 src_mcv: MCValue,
7117) InnerError!void {
7118 const pt = func.pt;
7119 const zcu = pt.zcu;
7120
7121 const abi_size: u32 = @intCast(ty.abiSize(zcu));
7122 const dst_ptr_mcv: MCValue = switch (base) {
7123 .reg => |base_reg| .{ .register_offset = .{ .reg = base_reg, .off = disp } },
7124 .frame => |base_frame_index| .{ .lea_frame = .{ .index = base_frame_index, .off = disp } },
7125 };
7126 switch (src_mcv) {
7127 .none,
7128 .unreach,
7129 .dead,
7130 .reserved_frame,
7131 => unreachable,
7132 .undef => |sym_index| {
7133 if (sym_index) |index| {
7134 return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } });
7135 }
7136
7137 try func.genInlineMemset(
7138 dst_ptr_mcv,
7139 src_mcv,
7140 .{ .immediate = abi_size },
7141 );
7142 },
7143 .register_offset,
7144 .memory,
7145 .indirect,
7146 .load_frame,
7147 .lea_frame,
7148 .load_symbol,
7149 .lea_symbol,
7150 => switch (abi_size) {
7151 0 => {},
7152 1, 2, 4, 8 => {
7153 const reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary);
7154 const src_lock = func.register_manager.lockRegAssumeUnused(reg);
7155 defer func.register_manager.unlockReg(src_lock);
7156
7157 try func.genSetReg(ty, reg, src_mcv);
7158 try func.genSetMem(base, disp, ty, .{ .register = reg });
7159 },
7160 else => try func.genInlineMemcpy(
7161 dst_ptr_mcv,
7162 src_mcv.address(),
7163 .{ .immediate = abi_size },
7164 ),
7165 },
7166 .register => |reg| {
7167 if (reg.class() == .vector) {
7168 const addr_reg = try func.copyToTmpRegister(Type.u64, dst_ptr_mcv);
7169
7170 const num_elem = ty.vectorLen(zcu);
7171 const elem_size = ty.childType(zcu).bitSize(zcu);
7172
7173 try func.setVl(.zero, num_elem, .{
7174 .vsew = switch (elem_size) {
7175 8 => .@"8",
7176 16 => .@"16",
7177 32 => .@"32",
7178 64 => .@"64",
7179 else => unreachable,
7180 },
7181 .vlmul = .m1,
7182 .vma = true,
7183 .vta = true,
7184 });
7185
7186 _ = try func.addInst(.{
7187 .tag = .pseudo_store_rm,
7188 .data = .{ .rm = .{
7189 .r = reg,
7190 .m = .{
7191 .base = .{ .reg = addr_reg },
7192 .mod = .{
7193 .disp = 0,
7194 .size = func.memSize(ty.childType(zcu)),
7195 .unsigned = false,
7196 },
7197 },
7198 } },
7199 });
7200
7201 return;
7202 }
7203
7204 const mem_size = switch (base) {
7205 .frame => |base_fi| mem_size: {
7206 assert(disp >= 0);
7207 const frame_abi_size = func.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)];
7208 const frame_spill_pad = func.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)];
7209 assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
7210 break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
7211 frame_abi_size
7212 else
7213 abi_size;
7214 },
7215 else => abi_size,
7216 };
7217 const src_size = math.ceilPowerOfTwoAssert(u32, abi_size);
7218 const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size));
7219 if (src_size > mem_size) {
7220 const frame_index = try func.allocFrameIndex(FrameAlloc.init(.{
7221 .size = src_size,
7222 .alignment = src_align,
7223 }));
7224 const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
7225 _ = try func.addInst(.{
7226 .tag = .pseudo_store_rm,
7227 .data = .{ .rm = .{
7228 .r = reg,
7229 .m = .{
7230 .base = .{ .frame = frame_index },
7231 .mod = .{
7232 .size = Memory.Size.fromByteSize(src_size),
7233 .unsigned = false,
7234 },
7235 },
7236 } },
7237 });
7238 try func.genSetMem(base, disp, ty, frame_mcv);
7239 try func.freeValue(frame_mcv);
7240 } else _ = try func.addInst(.{
7241 .tag = .pseudo_store_rm,
7242 .data = .{ .rm = .{
7243 .r = reg,
7244 .m = .{
7245 .base = base,
7246 .mod = .{
7247 .size = func.memSize(ty),
7248 .disp = disp,
7249 .unsigned = false,
7250 },
7251 },
7252 } },
7253 });
7254 },
7255 .register_pair => |src_regs| {
7256 var part_disp: i32 = disp;
7257 for (try func.splitType(ty), src_regs) |src_ty, src_reg| {
7258 try func.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
7259 part_disp += @intCast(src_ty.abiSize(zcu));
7260 }
7261 },
7262 .immediate => {
7263 // TODO: remove this lock in favor of a copyToTmpRegister when we load 64 bit immediates with
7264 // a register allocation.
7265 const reg, const reg_lock = try func.promoteReg(ty, src_mcv);
7266 defer if (reg_lock) |lock| func.register_manager.unlockReg(lock);
7267
7268 return func.genSetMem(base, disp, ty, .{ .register = reg });
7269 },
7270 .air_ref => |src_ref| try func.genSetMem(base, disp, ty, try func.resolveInst(src_ref)),
7271 }
7272}
7273
7274fn airBitCast(func: *Func, inst: Air.Inst.Index) !void {
7275 const pt = func.pt;
7276 const zcu = pt.zcu;
7277
7278 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7279 const result = if (func.liveness.isUnused(inst)) .unreach else result: {
7280 const src_mcv = try func.resolveInst(ty_op.operand);
7281
7282 const src_ty = func.typeOf(ty_op.operand);
7283 if (src_ty.toIntern() == .bool_type) break :result src_mcv;
7284 const dst_ty = func.typeOfIndex(inst);
7285
7286 const src_lock = if (src_mcv.getReg()) |reg| func.register_manager.lockReg(reg) else null;
7287 defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7288
7289 const dst_mcv = if (dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and src_mcv != .register_pair and
7290 func.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
7291 const dst_mcv = try func.allocRegOrMem(dst_ty, inst, true);
7292 try func.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) {
7293 .lt => dst_ty,
7294 .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
7295 .gt => src_ty,
7296 }, dst_mcv, src_mcv);
7297 break :dst dst_mcv;
7298 };
7299
7300 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and
7301 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
7302
7303 const abi_size = dst_ty.abiSize(zcu);
7304 const bit_size = dst_ty.bitSize(zcu);
7305 if (abi_size * 8 <= bit_size) break :result dst_mcv;
7306
7307 return func.fail("TODO: airBitCast {f} to {f}", .{ src_ty.fmt(pt), dst_ty.fmt(pt) });
7308 };
7309 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7310}
7311
7312fn airArrayToSlice(func: *Func, inst: Air.Inst.Index) !void {
7313 const pt = func.pt;
7314 const zcu = pt.zcu;
7315 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7316
7317 const slice_ty = func.typeOfIndex(inst);
7318 const ptr_ty = func.typeOf(ty_op.operand);
7319 const ptr = try func.resolveInst(ty_op.operand);
7320 const array_ty = ptr_ty.childType(zcu);
7321 const array_len = array_ty.arrayLen(zcu);
7322
7323 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(slice_ty, zcu));
7324 try func.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
7325 try func.genSetMem(
7326 .{ .frame = frame_index },
7327 @intCast(ptr_ty.abiSize(zcu)),
7328 Type.u64,
7329 .{ .immediate = array_len },
7330 );
7331
7332 const result = MCValue{ .load_frame = .{ .index = frame_index } };
7333 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7334}
7335
7336fn airFloatFromInt(func: *Func, inst: Air.Inst.Index) !void {
7337 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7338 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7339 const pt = func.pt;
7340 const zcu = pt.zcu;
7341
7342 const operand = try func.resolveInst(ty_op.operand);
7343
7344 const src_ty = func.typeOf(ty_op.operand);
7345 const dst_ty = ty_op.ty.toType();
7346
7347 const src_reg, const src_lock = try func.promoteReg(src_ty, operand);
7348 defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7349
7350 const is_unsigned = dst_ty.isUnsignedInt(zcu);
7351 const src_bits = src_ty.bitSize(zcu);
7352 const dst_bits = dst_ty.bitSize(zcu);
7353
7354 switch (src_bits) {
7355 32, 64 => {},
7356 else => try func.truncateRegister(src_ty, src_reg),
7357 }
7358
7359 const int_zcu: Mir.FcvtOp = switch (src_bits) {
7360 8, 16, 32 => if (is_unsigned) .wu else .w,
7361 64 => if (is_unsigned) .lu else .l,
7362 else => return func.fail("TODO: airFloatFromInt src size: {d}", .{src_bits}),
7363 };
7364
7365 const float_zcu: enum { s, d } = switch (dst_bits) {
7366 32 => .s,
7367 64 => .d,
7368 else => return func.fail("TODO: airFloatFromInt dst size {d}", .{dst_bits}),
7369 };
7370
7371 const dst_reg, const dst_lock = try func.allocReg(.int);
7372 defer func.register_manager.unlockReg(dst_lock);
7373
7374 _ = try func.addInst(.{
7375 .tag = switch (float_zcu) {
7376 .s => switch (int_zcu) {
7377 .l => .fcvtsl,
7378 .lu => .fcvtslu,
7379 .w => .fcvtsw,
7380 .wu => .fcvtswu,
7381 },
7382 .d => switch (int_zcu) {
7383 .l => .fcvtdl,
7384 .lu => .fcvtdlu,
7385 .w => .fcvtdw,
7386 .wu => .fcvtdwu,
7387 },
7388 },
7389 .data = .{ .rr = .{
7390 .rd = dst_reg,
7391 .rs = src_reg,
7392 } },
7393 });
7394
7395 break :result .{ .register = dst_reg };
7396 };
7397 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7398}
7399
7400fn airIntFromFloat(func: *Func, inst: Air.Inst.Index) !void {
7401 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7402 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7403 const pt = func.pt;
7404 const zcu = pt.zcu;
7405
7406 const operand = try func.resolveInst(ty_op.operand);
7407 const src_ty = func.typeOf(ty_op.operand);
7408 const dst_ty = ty_op.ty.toType();
7409
7410 const is_unsigned = dst_ty.isUnsignedInt(zcu);
7411 const src_bits = src_ty.bitSize(zcu);
7412 const dst_bits = dst_ty.bitSize(zcu);
7413
7414 const float_zcu: enum { s, d } = switch (src_bits) {
7415 32 => .s,
7416 64 => .d,
7417 else => return func.fail("TODO: airIntFromFloat src size {d}", .{src_bits}),
7418 };
7419
7420 const int_zcu: Mir.FcvtOp = switch (dst_bits) {
7421 32 => if (is_unsigned) .wu else .w,
7422 8, 16, 64 => if (is_unsigned) .lu else .l,
7423 else => return func.fail("TODO: airIntFromFloat dst size: {d}", .{dst_bits}),
7424 };
7425
7426 const src_reg, const src_lock = try func.promoteReg(src_ty, operand);
7427 defer if (src_lock) |lock| func.register_manager.unlockReg(lock);
7428
7429 const dst_reg, const dst_lock = try func.allocReg(.int);
7430 defer func.register_manager.unlockReg(dst_lock);
7431
7432 _ = try func.addInst(.{
7433 .tag = switch (float_zcu) {
7434 .s => switch (int_zcu) {
7435 .l => .fcvtls,
7436 .lu => .fcvtlus,
7437 .w => .fcvtws,
7438 .wu => .fcvtwus,
7439 },
7440 .d => switch (int_zcu) {
7441 .l => .fcvtld,
7442 .lu => .fcvtlud,
7443 .w => .fcvtwd,
7444 .wu => .fcvtwud,
7445 },
7446 },
7447 .data = .{ .rr = .{
7448 .rd = dst_reg,
7449 .rs = src_reg,
7450 } },
7451 });
7452
7453 break :result .{ .register = dst_reg };
7454 };
7455 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7456}
7457
7458fn airCmpxchg(func: *Func, inst: Air.Inst.Index, strength: enum { weak, strong }) !void {
7459 _ = strength; // TODO: do something with this
7460
7461 const pt = func.pt;
7462 const zcu = pt.zcu;
7463 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7464 const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
7465
7466 const ptr_ty = func.typeOf(extra.ptr);
7467 const val_ty = func.typeOf(extra.expected_value);
7468 const val_abi_size: u32 = @intCast(val_ty.abiSize(pt.zcu));
7469
7470 switch (val_abi_size) {
7471 1, 2, 4, 8 => {},
7472 else => return func.fail("TODO: airCmpxchg Int size {}", .{val_abi_size}),
7473 }
7474
7475 const lr_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.successOrder()) {
7476 .unordered,
7477 => unreachable,
7478
7479 .monotonic,
7480 .release,
7481 => .{ .aq = .none, .rl = .none },
7482 .acquire,
7483 .acq_rel,
7484 => .{ .aq = .aq, .rl = .none },
7485 .seq_cst => .{ .aq = .aq, .rl = .rl },
7486 };
7487
7488 const sc_order: struct { aq: Mir.Barrier, rl: Mir.Barrier } = switch (extra.failureOrder()) {
7489 .unordered,
7490 .release,
7491 .acq_rel,
7492 => unreachable,
7493
7494 .monotonic,
7495 .acquire,
7496 .seq_cst,
7497 => switch (extra.successOrder()) {
7498 .release,
7499 .seq_cst,
7500 => .{ .aq = .none, .rl = .rl },
7501 else => .{ .aq = .none, .rl = .none },
7502 },
7503 };
7504
7505 const ptr_mcv = try func.resolveInst(extra.ptr);
7506 const ptr_reg, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7507 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
7508
7509 const exp_mcv = try func.resolveInst(extra.expected_value);
7510 const exp_reg, const exp_lock = try func.promoteReg(val_ty, exp_mcv);
7511 defer if (exp_lock) |lock| func.register_manager.unlockReg(lock);
7512 try func.truncateRegister(val_ty, exp_reg);
7513
7514 const new_mcv = try func.resolveInst(extra.new_value);
7515 const new_reg, const new_lock = try func.promoteReg(val_ty, new_mcv);
7516 defer if (new_lock) |lock| func.register_manager.unlockReg(lock);
7517 try func.truncateRegister(val_ty, new_reg);
7518
7519 const branch_reg, const branch_lock = try func.allocReg(.int);
7520 defer func.register_manager.unlockReg(branch_lock);
7521
7522 const fallthrough_reg, const fallthrough_lock = try func.allocReg(.int);
7523 defer func.register_manager.unlockReg(fallthrough_lock);
7524
7525 const jump_back = try func.addInst(.{
7526 .tag = if (val_ty.bitSize(zcu) <= 32) .lrw else .lrd,
7527 .data = .{ .amo = .{
7528 .aq = lr_order.aq,
7529 .rl = lr_order.rl,
7530 .rd = branch_reg,
7531 .rs1 = ptr_reg,
7532 .rs2 = .zero,
7533 } },
7534 });
7535 try func.truncateRegister(val_ty, branch_reg);
7536
7537 const jump_forward = try func.addInst(.{
7538 .tag = .bne,
7539 .data = .{ .b_type = .{
7540 .rs1 = branch_reg,
7541 .rs2 = exp_reg,
7542 .inst = undefined,
7543 } },
7544 });
7545
7546 _ = try func.addInst(.{
7547 .tag = if (val_ty.bitSize(zcu) <= 32) .scw else .scd,
7548 .data = .{ .amo = .{
7549 .aq = sc_order.aq,
7550 .rl = sc_order.rl,
7551 .rd = fallthrough_reg,
7552 .rs1 = ptr_reg,
7553 .rs2 = new_reg,
7554 } },
7555 });
7556 try func.truncateRegister(Type.bool, fallthrough_reg);
7557
7558 _ = try func.addInst(.{
7559 .tag = .bne,
7560 .data = .{ .b_type = .{
7561 .rs1 = fallthrough_reg,
7562 .rs2 = .zero,
7563 .inst = jump_back,
7564 } },
7565 });
7566
7567 func.performReloc(jump_forward);
7568
7569 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7570 const dst_mcv = try func.allocRegOrMem(func.typeOfIndex(inst), inst, false);
7571
7572 const tmp_reg, const tmp_lock = try func.allocReg(.int);
7573 defer func.register_manager.unlockReg(tmp_lock);
7574
7575 try func.genBinOp(
7576 .cmp_neq,
7577 .{ .register = branch_reg },
7578 val_ty,
7579 .{ .register = exp_reg },
7580 val_ty,
7581 tmp_reg,
7582 );
7583
7584 try func.genCopy(val_ty, dst_mcv, .{ .register = branch_reg });
7585 try func.genCopy(
7586 Type.bool,
7587 dst_mcv.address().offset(@intCast(val_abi_size)).deref(),
7588 .{ .register = tmp_reg },
7589 );
7590
7591 break :result dst_mcv;
7592 };
7593
7594 return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
7595}
7596
7597fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
7598 const pt = func.pt;
7599 const zcu = pt.zcu;
7600 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7601 const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
7602
7603 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7604 const op = extra.op();
7605 const order = extra.ordering();
7606
7607 const ptr_ty = func.typeOf(pl_op.operand);
7608 const ptr_mcv = try func.resolveInst(pl_op.operand);
7609
7610 const val_ty = func.typeOf(extra.operand);
7611 const val_size = val_ty.abiSize(zcu);
7612 const val_mcv = try func.resolveInst(extra.operand);
7613
7614 if (!math.isPowerOfTwo(val_size))
7615 return func.fail("TODO: airAtomicRmw non-pow 2", .{});
7616
7617 switch (val_ty.zigTypeTag(pt.zcu)) {
7618 .@"enum", .int => {},
7619 inline .bool, .float, .pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}),
7620 else => unreachable,
7621 }
7622
7623 const method: enum { amo, loop } = switch (val_size) {
7624 1, 2 => .loop,
7625 4, 8 => .amo,
7626 else => unreachable,
7627 };
7628
7629 const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
7630 defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
7631
7632 const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv);
7633 defer if (val_lock) |lock| func.register_manager.unlockReg(lock);
7634
7635 const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
7636 assert(result_mcv == .register); // should fit into 8 bytes
7637 const result_reg = result_mcv.register;
7638
7639 const aq, const rl = switch (order) {
7640 .unordered => unreachable,
7641 .monotonic => .{ false, false },
7642 .acquire => .{ true, false },
7643 .release => .{ false, true },
7644 .acq_rel => .{ true, true },
7645 .seq_cst => .{ true, true },
7646 };
7647
7648 switch (method) {
7649 .amo => {
7650 const is_d = val_ty.abiSize(zcu) == 8;
7651 const is_un = val_ty.isUnsignedInt(zcu);
7652
7653 const mnem: Mnemonic = switch (op) {
7654 // zig fmt: off
7655 .Xchg => if (is_d) .amoswapd else .amoswapw,
7656 .Add => if (is_d) .amoaddd else .amoaddw,
7657 .And => if (is_d) .amoandd else .amoandw,
7658 .Or => if (is_d) .amoord else .amoorw,
7659 .Xor => if (is_d) .amoxord else .amoxorw,
7660 .Max => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
7661 .Min => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
7662 else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),
7663 // zig fmt: on
7664 };
7665
7666 _ = try func.addInst(.{
7667 .tag = mnem,
7668 .data = .{ .amo = .{
7669 .rd = result_reg,
7670 .rs1 = ptr_register,
7671 .rs2 = val_register,
7672 .aq = if (aq) .aq else .none,
7673 .rl = if (rl) .rl else .none,
7674 } },
7675 });
7676 },
7677 .loop => {
7678 // where we'll jump back when the sc fails
7679 const jump_back = try func.addInst(.{
7680 .tag = .lrw,
7681 .data = .{ .amo = .{
7682 .rd = result_reg,
7683 .rs1 = ptr_register,
7684 .rs2 = .zero,
7685 .aq = if (aq) .aq else .none,
7686 .rl = if (rl) .rl else .none,
7687 } },
7688 });
7689
7690 const after_reg, const after_lock = try func.allocReg(.int);
7691 defer func.register_manager.unlockReg(after_lock);
7692
7693 switch (op) {
7694 .Add, .Sub => |tag| {
7695 _ = try func.genBinOp(
7696 switch (tag) {
7697 .Add => .add,
7698 .Sub => .sub,
7699 else => unreachable,
7700 },
7701 .{ .register = result_reg },
7702 val_ty,
7703 .{ .register = val_register },
7704 val_ty,
7705 after_reg,
7706 );
7707 },
7708
7709 else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
7710 }
7711
7712 _ = try func.addInst(.{
7713 .tag = .scw,
7714 .data = .{ .amo = .{
7715 .rd = after_reg,
7716 .rs1 = ptr_register,
7717 .rs2 = after_reg,
7718 .aq = if (aq) .aq else .none,
7719 .rl = if (rl) .rl else .none,
7720 } },
7721 });
7722
7723 _ = try func.addInst(.{
7724 .tag = .bne,
7725 .data = .{ .b_type = .{
7726 .inst = jump_back,
7727 .rs1 = after_reg,
7728 .rs2 = .zero,
7729 } },
7730 });
7731 },
7732 }
7733 break :result result_mcv;
7734 };
7735
7736 return func.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
7737}
7738
7739fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
7740 const pt = func.pt;
7741 const zcu = pt.zcu;
7742 const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
7743 const order: std.builtin.AtomicOrder = atomic_load.order;
7744
7745 const ptr_ty = func.typeOf(atomic_load.ptr);
7746 const elem_ty = ptr_ty.childType(zcu);
7747 const ptr_mcv = try func.resolveInst(atomic_load.ptr);
7748
7749 const bit_size = elem_ty.bitSize(zcu);
7750 if (bit_size > 64) return func.fail("TODO: airAtomicLoad > 64 bits", .{});
7751
7752 const result_mcv: MCValue = if (func.liveness.isUnused(inst))
7753 .{ .register = .zero }
7754 else
7755 try func.allocRegOrMem(elem_ty, inst, true);
7756 assert(result_mcv == .register); // should be less than 8 bytes
7757
7758 if (order == .seq_cst) {
7759 _ = try func.addInst(.{
7760 .tag = .fence,
7761 .data = .{ .fence = .{
7762 .pred = .rw,
7763 .succ = .rw,
7764 } },
7765 });
7766 }
7767
7768 try func.load(result_mcv, ptr_mcv, ptr_ty);
7769
7770 switch (order) {
7771 // Don't guarantee other memory operations to be ordered after the load.
7772 .unordered, .monotonic => {},
7773 // Make sure all previous reads happen before any reading or writing occurs.
7774 .acquire, .seq_cst => {
7775 _ = try func.addInst(.{
7776 .tag = .fence,
7777 .data = .{ .fence = .{
7778 .pred = .r,
7779 .succ = .rw,
7780 } },
7781 });
7782 },
7783 else => unreachable,
7784 }
7785
7786 return func.finishAir(inst, result_mcv, .{ atomic_load.ptr, .none, .none });
7787}
7788
7789fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
7790 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
7791
7792 const ptr_ty = func.typeOf(bin_op.lhs);
7793 const ptr_mcv = try func.resolveInst(bin_op.lhs);
7794
7795 const val_ty = func.typeOf(bin_op.rhs);
7796 const val_mcv = try func.resolveInst(bin_op.rhs);
7797
7798 const bit_size = val_ty.bitSize(func.pt.zcu);
7799 if (bit_size > 64) return func.fail("TODO: airAtomicStore > 64 bits", .{});
7800
7801 switch (order) {
7802 .unordered, .monotonic => {},
7803 .release, .seq_cst => {
7804 _ = try func.addInst(.{
7805 .tag = .fence,
7806 .data = .{ .fence = .{
7807 .pred = .rw,
7808 .succ = .w,
7809 } },
7810 });
7811 },
7812 else => unreachable,
7813 }
7814
7815 try func.store(ptr_mcv, val_mcv, ptr_ty);
7816
7817 if (order == .seq_cst) {
7818 _ = try func.addInst(.{
7819 .tag = .fence,
7820 .data = .{ .fence = .{
7821 .pred = .rw,
7822 .succ = .rw,
7823 } },
7824 });
7825 }
7826
7827 return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7828}
7829
7830fn airMemset(func: *Func, inst: Air.Inst.Index, safety: bool) !void {
7831 const pt = func.pt;
7832 const zcu = pt.zcu;
7833 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
7834
7835 result: {
7836 if (!safety and (try func.resolveInst(bin_op.rhs)) == .undef) break :result;
7837
7838 const dst_ptr = try func.resolveInst(bin_op.lhs);
7839 const dst_ptr_ty = func.typeOf(bin_op.lhs);
7840 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
7841 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7842 else => null,
7843 };
7844 defer if (dst_ptr_lock) |lock| func.register_manager.unlockReg(lock);
7845
7846 const src_val = try func.resolveInst(bin_op.rhs);
7847 const elem_ty = func.typeOf(bin_op.rhs);
7848 const src_val_lock: ?RegisterLock = switch (src_val) {
7849 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7850 else => null,
7851 };
7852 defer if (src_val_lock) |lock| func.register_manager.unlockReg(lock);
7853
7854 const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu));
7855
7856 if (elem_abi_size == 1) {
7857 const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
7858 // TODO: this only handles slices stored in the stack
7859 .slice => dst_ptr,
7860 .one => dst_ptr,
7861 .c, .many => unreachable,
7862 };
7863 const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) {
7864 // TODO: this only handles slices stored in the stack
7865 .slice => dst_ptr.address().offset(8).deref(),
7866 .one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) },
7867 .c, .many => unreachable,
7868 };
7869 const len_lock: ?RegisterLock = switch (len) {
7870 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7871 else => null,
7872 };
7873 defer if (len_lock) |lock| func.register_manager.unlockReg(lock);
7874
7875 try func.genInlineMemset(ptr, src_val, len);
7876 break :result;
7877 }
7878
7879 // Store the first element, and then rely on memcpy copying forwards.
7880 // Length zero requires a runtime check - so we handle arrays specially
7881 // here to elide it.
7882 switch (dst_ptr_ty.ptrSize(zcu)) {
7883 .slice => return func.fail("TODO: airMemset Slices", .{}),
7884 .one => {
7885 const elem_ptr_ty = try pt.singleMutPtrType(elem_ty);
7886
7887 const len = dst_ptr_ty.childType(zcu).arrayLen(zcu);
7888
7889 assert(len != 0); // prevented by Sema
7890 try func.store(dst_ptr, src_val, elem_ptr_ty);
7891
7892 const second_elem_ptr_reg, const second_elem_ptr_lock = try func.allocReg(.int);
7893 defer func.register_manager.unlockReg(second_elem_ptr_lock);
7894
7895 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
7896
7897 try func.genSetReg(Type.u64, second_elem_ptr_reg, .{ .register_offset = .{
7898 .reg = try func.copyToTmpRegister(Type.u64, dst_ptr),
7899 .off = elem_abi_size,
7900 } });
7901
7902 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
7903 try func.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy);
7904 },
7905 .c, .many => unreachable,
7906 }
7907 }
7908 return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7909}
7910
7911fn airMemcpy(func: *Func, inst: Air.Inst.Index) !void {
7912 const pt = func.pt;
7913 const zcu = pt.zcu;
7914 const bin_op = func.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
7915
7916 const dst_ptr = try func.resolveInst(bin_op.lhs);
7917 const src_ptr = try func.resolveInst(bin_op.rhs);
7918
7919 const dst_ty = func.typeOf(bin_op.lhs);
7920
7921 const len_mcv: MCValue = switch (dst_ty.ptrSize(zcu)) {
7922 .slice => len: {
7923 const len_reg, const len_lock = try func.allocReg(.int);
7924 defer func.register_manager.unlockReg(len_lock);
7925
7926 const elem_size = dst_ty.childType(zcu).abiSize(zcu);
7927 try func.genBinOp(
7928 .mul,
7929 .{ .immediate = elem_size },
7930 Type.u64,
7931 dst_ptr.address().offset(8).deref(),
7932 Type.u64,
7933 len_reg,
7934 );
7935 break :len .{ .register = len_reg };
7936 },
7937 .one => len: {
7938 const array_ty = dst_ty.childType(zcu);
7939 break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) };
7940 },
7941 else => |size| return func.fail("TODO: airMemcpy size {s}", .{@tagName(size)}),
7942 };
7943 const len_lock: ?RegisterLock = switch (len_mcv) {
7944 .register => |reg| func.register_manager.lockRegAssumeUnused(reg),
7945 else => null,
7946 };
7947 defer if (len_lock) |lock| func.register_manager.unlockReg(lock);
7948
7949 try func.genInlineMemcpy(dst_ptr, src_ptr, len_mcv);
7950
7951 return func.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
7952}
7953
7954fn airMemmove(func: *Func, inst: Air.Inst.Index) !void {
7955 _ = inst;
7956 return func.fail("TODO implement airMemmove for riscv64", .{});
7957}
7958
7959fn airTagName(func: *Func, inst: Air.Inst.Index) !void {
7960 const pt = func.pt;
7961
7962 const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7963 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: {
7964 const enum_ty = func.typeOf(un_op);
7965
7966 // TODO: work out the bugs
7967 if (true) return func.fail("TODO: airTagName", .{});
7968
7969 const param_regs = abi.Registers.Integer.function_arg_regs;
7970 const dst_mcv = try func.allocRegOrMem(Type.u64, inst, false);
7971 try func.genSetReg(Type.u64, param_regs[0], dst_mcv.address());
7972
7973 const operand = try func.resolveInst(un_op);
7974 try func.genSetReg(enum_ty, param_regs[1], operand);
7975
7976 const lazy_sym: link.File.LazySymbol = .{ .kind = .code, .ty = enum_ty.toIntern() };
7977 const elf_file = func.bin_file.cast(link.File.Elf).?;
7978 const zo = elf_file.zigObjectPtr().?;
7979 const sym_index = zo.getOrCreateMetadataForLazySymbol(elf_file, pt, lazy_sym) catch |err|
7980 return func.fail("{s} creating lazy symbol", .{@errorName(err)});
7981
7982 if (func.mod.pic) {
7983 return func.fail("TODO: airTagName pic", .{});
7984 } else {
7985 try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } });
7986 _ = try func.addInst(.{
7987 .tag = .jalr,
7988 .data = .{ .i_type = .{
7989 .rd = .ra,
7990 .rs1 = .ra,
7991 .imm12 = Immediate.s(0),
7992 } },
7993 });
7994 }
7995
7996 break :result dst_mcv;
7997 };
7998 return func.finishAir(inst, result, .{ un_op, .none, .none });
7999}
8000
8001fn airErrorName(func: *Func, inst: Air.Inst.Index) !void {
8002 _ = inst;
8003 return func.fail("TODO: airErrorName", .{});
8004}
8005
8006fn airSplat(func: *Func, inst: Air.Inst.Index) !void {
8007 const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8008 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSplat for riscv64", .{});
8009 return func.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8010}
8011
8012fn airSelect(func: *Func, inst: Air.Inst.Index) !void {
8013 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8014 const extra = func.air.extraData(Air.Bin, pl_op.payload).data;
8015 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airSelect for riscv64", .{});
8016 return func.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
8017}
8018
8019fn airShuffleOne(func: *Func, inst: Air.Inst.Index) !void {
8020 _ = inst;
8021 return func.fail("TODO implement airShuffleOne for riscv64", .{});
8022}
8023
8024fn airShuffleTwo(func: *Func, inst: Air.Inst.Index) !void {
8025 _ = inst;
8026 return func.fail("TODO implement airShuffleTwo for riscv64", .{});
8027}
8028
8029fn airReduce(func: *Func, inst: Air.Inst.Index) !void {
8030 const reduce = func.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
8031 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airReduce for riscv64", .{});
8032 return func.finishAir(inst, result, .{ reduce.operand, .none, .none });
8033}
8034
8035fn airAggregateInit(func: *Func, inst: Air.Inst.Index) !void {
8036 const pt = func.pt;
8037 const zcu = pt.zcu;
8038 const result_ty = func.typeOfIndex(inst);
8039 const len: usize = @intCast(result_ty.arrayLen(zcu));
8040 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8041 const elements: []const Air.Inst.Ref = @ptrCast(func.air.extra.items[ty_pl.payload..][0..len]);
8042
8043 const result: MCValue = result: {
8044 switch (result_ty.zigTypeTag(zcu)) {
8045 .@"struct" => {
8046 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
8047 if (result_ty.containerLayout(zcu) == .@"packed") {
8048 const struct_obj = zcu.typeToStruct(result_ty).?;
8049 try func.genInlineMemset(
8050 .{ .lea_frame = .{ .index = frame_index } },
8051 .{ .immediate = 0 },
8052 .{ .immediate = result_ty.abiSize(zcu) },
8053 );
8054
8055 for (elements, 0..) |elem, elem_i_usize| {
8056 const elem_i: u32 = @intCast(elem_i_usize);
8057 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
8058
8059 const elem_ty = result_ty.fieldType(elem_i, zcu);
8060 const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
8061 if (elem_bit_size > 64) {
8062 return func.fail(
8063 "TODO airAggregateInit implement packed structs with large fields",
8064 .{},
8065 );
8066 }
8067
8068 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
8069 const elem_abi_bits = elem_abi_size * 8;
8070 const elem_off = zcu.structPackedFieldBitOffset(struct_obj, elem_i);
8071 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
8072 const elem_bit_off = elem_off % elem_abi_bits;
8073 const elem_mcv = try func.resolveInst(elem);
8074
8075 _ = elem_byte_off;
8076 _ = elem_bit_off;
8077
8078 const elem_lock = switch (elem_mcv) {
8079 .register => |reg| func.register_manager.lockReg(reg),
8080 .immediate => |imm| lock: {
8081 if (imm == 0) continue;
8082 break :lock null;
8083 },
8084 else => null,
8085 };
8086 defer if (elem_lock) |lock| func.register_manager.unlockReg(lock);
8087
8088 return func.fail("TODO: airAggregateInit packed structs", .{});
8089 }
8090 } else for (elements, 0..) |elem, elem_i| {
8091 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
8092
8093 const elem_ty = result_ty.fieldType(elem_i, zcu);
8094 const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
8095 const elem_mcv = try func.resolveInst(elem);
8096 try func.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv);
8097 }
8098 break :result .{ .load_frame = .{ .index = frame_index } };
8099 },
8100 .array => {
8101 const elem_ty = result_ty.childType(zcu);
8102 const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(result_ty, zcu));
8103 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
8104
8105 for (elements, 0..) |elem, elem_i| {
8106 const elem_mcv = try func.resolveInst(elem);
8107 const elem_off: i32 = @intCast(elem_size * elem_i);
8108 try func.genSetMem(
8109 .{ .frame = frame_index },
8110 elem_off,
8111 elem_ty,
8112 elem_mcv,
8113 );
8114 }
8115 if (result_ty.sentinel(zcu)) |sentinel| try func.genSetMem(
8116 .{ .frame = frame_index },
8117 @intCast(elem_size * elements.len),
8118 elem_ty,
8119 try func.genTypedValue(sentinel),
8120 );
8121 break :result .{ .load_frame = .{ .index = frame_index } };
8122 },
8123 else => return func.fail("TODO: airAggregate {f}", .{result_ty.fmt(pt)}),
8124 }
8125 };
8126
8127 if (elements.len <= Air.Liveness.bpi - 1) {
8128 var buf = [1]Air.Inst.Ref{.none} ** (Air.Liveness.bpi - 1);
8129 @memcpy(buf[0..elements.len], elements);
8130 return func.finishAir(inst, result, buf);
8131 }
8132 var bt = func.liveness.iterateBigTomb(inst);
8133 for (elements) |elem| try func.feed(&bt, elem);
8134 return func.finishAirResult(inst, result);
8135}
8136
8137fn airUnionInit(func: *Func, inst: Air.Inst.Index) !void {
8138 const ty_pl = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8139 const extra = func.air.extraData(Air.UnionInit, ty_pl.payload).data;
8140 _ = extra;
8141 return func.fail("TODO implement airUnionInit for riscv64", .{});
8142 // return func.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value });
8143}
8144
8145fn airPrefetch(func: *Func, inst: Air.Inst.Index) !void {
8146 const prefetch = func.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
8147 // TODO: RISC-V does have prefetch instruction variants.
8148 // see here: https://raw.githubusercontent.com/riscv/riscv-CMOs/master/specifications/cmobase-v1.0.1.pdf
8149 return func.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
8150}
8151
8152fn airMulAdd(func: *Func, inst: Air.Inst.Index) !void {
8153 const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8154 const extra = func.air.extraData(Air.Bin, pl_op.payload).data;
8155 const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else {
8156 return func.fail("TODO implement airMulAdd for riscv64", .{});
8157 };
8158 return func.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
8159}
8160
8161fn resolveInst(func: *Func, ref: Air.Inst.Ref) InnerError!MCValue {
8162 const pt = func.pt;
8163 const zcu = pt.zcu;
8164
8165 // If the type has no codegen bits, no need to store it.
8166 const inst_ty = func.typeOf(ref);
8167 if (!inst_ty.hasRuntimeBits(zcu))
8168 return .none;
8169
8170 const mcv = if (ref.toIndex()) |inst| mcv: {
8171 break :mcv func.inst_tracking.getPtr(inst).?.short;
8172 } else mcv: {
8173 const ip_index = ref.toInterned().?;
8174 const gop = try func.const_tracking.getOrPut(func.gpa, ip_index);
8175 if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(
8176 try func.genTypedValue(Value.fromInterned(ip_index)),
8177 );
8178 break :mcv gop.value_ptr.short;
8179 };
8180
8181 return mcv;
8182}
8183
8184fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking {
8185 const tracking = func.inst_tracking.getPtr(inst).?;
8186 return switch (tracking.short) {
8187 .none, .unreach, .dead => unreachable,
8188 else => tracking,
8189 };
8190}
8191
8192fn genTypedValue(func: *Func, val: Value) InnerError!MCValue {
8193 const pt = func.pt;
8194
8195 const lf = func.bin_file;
8196 const src_loc = func.src_loc;
8197
8198 const result: codegen.GenResult = if (val.isUndef(pt.zcu))
8199 switch (try lf.lowerUav(pt, val.toIntern(), .none, src_loc)) {
8200 .sym_index => |sym_index| .{ .mcv = .{ .load_symbol = sym_index } },
8201 .fail => |em| .{ .fail = em },
8202 }
8203 else
8204 try codegen.genTypedValue(lf, pt, src_loc, val, func.target);
8205 const mcv: MCValue = switch (result) {
8206 .mcv => |mcv| switch (mcv) {
8207 .none => .none,
8208 .undef => unreachable,
8209 .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } },
8210 .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } },
8211 .immediate => |imm| .{ .immediate = imm },
8212 .memory => |addr| .{ .memory = addr },
8213 .load_got, .load_direct, .lea_direct => {
8214 return func.fail("TODO: genTypedValue {s}", .{@tagName(mcv)});
8215 },
8216 },
8217 .fail => |msg| return func.failMsg(msg),
8218 };
8219 return mcv;
8220}
8221
8222const CallMCValues = struct {
8223 args: []MCValue,
8224 return_value: InstTracking,
8225 stack_byte_count: u31,
8226 stack_align: Alignment,
8227
8228 fn deinit(call: *CallMCValues, func: *Func) void {
8229 func.gpa.free(call.args);
8230 call.* = undefined;
8231 }
8232};
8233
8234/// Caller must call `CallMCValues.deinit`.
8235fn resolveCallingConventionValues(
8236 func: *Func,
8237 fn_info: InternPool.Key.FuncType,
8238 var_args: []const Type,
8239) !CallMCValues {
8240 const pt = func.pt;
8241 const zcu = pt.zcu;
8242 const ip = &zcu.intern_pool;
8243
8244 const param_types = try func.gpa.alloc(Type, fn_info.param_types.len + var_args.len);
8245 defer func.gpa.free(param_types);
8246
8247 for (param_types[0..fn_info.param_types.len], fn_info.param_types.get(ip)) |*dest, src| {
8248 dest.* = Type.fromInterned(src);
8249 }
8250 for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty|
8251 param_ty.* = func.promoteVarArg(arg_ty);
8252
8253 const cc = fn_info.cc;
8254 var result: CallMCValues = .{
8255 .args = try func.gpa.alloc(MCValue, param_types.len),
8256 // These undefined values must be populated before returning from this function.
8257 .return_value = undefined,
8258 .stack_byte_count = 0,
8259 .stack_align = undefined,
8260 };
8261 errdefer func.gpa.free(result.args);
8262
8263 const ret_ty = Type.fromInterned(fn_info.return_type);
8264
8265 switch (cc) {
8266 .naked => {
8267 assert(result.args.len == 0);
8268 result.return_value = InstTracking.init(.unreach);
8269 result.stack_align = .@"8";
8270 },
8271 .riscv64_lp64, .auto => {
8272 if (result.args.len > 8) {
8273 return func.fail("RISC-V calling convention does not support more than 8 arguments", .{});
8274 }
8275
8276 var ret_int_reg_i: u32 = 0;
8277 var param_int_reg_i: u32 = 0;
8278
8279 result.stack_align = .@"16";
8280
8281 // Return values
8282 if (ret_ty.zigTypeTag(zcu) == .noreturn) {
8283 result.return_value = InstTracking.init(.unreach);
8284 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
8285 result.return_value = InstTracking.init(.none);
8286 } else {
8287 var ret_tracking: [2]InstTracking = undefined;
8288 var ret_tracking_i: usize = 0;
8289 var ret_float_reg_i: usize = 0;
8290
8291 const classes = mem.sliceTo(&abi.classifySystem(ret_ty, zcu), .none);
8292
8293 for (classes) |class| switch (class) {
8294 .integer => {
8295 const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i];
8296 ret_int_reg_i += 1;
8297
8298 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_int_reg });
8299 ret_tracking_i += 1;
8300 },
8301 .float => {
8302 const ret_float_reg = abi.Registers.Float.function_ret_regs[ret_float_reg_i];
8303 ret_float_reg_i += 1;
8304
8305 ret_tracking[ret_tracking_i] = InstTracking.init(.{ .register = ret_float_reg });
8306 ret_tracking_i += 1;
8307 },
8308 .memory => {
8309 const ret_int_reg = abi.Registers.Integer.function_ret_regs[ret_int_reg_i];
8310 ret_int_reg_i += 1;
8311 const ret_indirect_reg = abi.Registers.Integer.function_arg_regs[param_int_reg_i];
8312 param_int_reg_i += 1;
8313
8314 ret_tracking[ret_tracking_i] = .{
8315 .short = .{ .indirect = .{ .reg = ret_int_reg } },
8316 .long = .{ .indirect = .{ .reg = ret_indirect_reg } },
8317 };
8318 ret_tracking_i += 1;
8319 },
8320 else => return func.fail("TODO: C calling convention return class {}", .{class}),
8321 };
8322
8323 result.return_value = switch (ret_tracking_i) {
8324 else => return func.fail("ty {f} took {} tracking return indices", .{ ret_ty.fmt(pt), ret_tracking_i }),
8325 1 => ret_tracking[0],
8326 2 => InstTracking.init(.{ .register_pair = .{
8327 ret_tracking[0].short.register, ret_tracking[1].short.register,
8328 } }),
8329 };
8330 }
8331
8332 var param_float_reg_i: usize = 0;
8333
8334 for (param_types, result.args) |ty, *arg| {
8335 if (!ty.hasRuntimeBitsIgnoreComptime(zcu)) {
8336 assert(cc == .auto);
8337 arg.* = .none;
8338 continue;
8339 }
8340
8341 var arg_mcv: [2]MCValue = undefined;
8342 var arg_mcv_i: usize = 0;
8343
8344 const classes = mem.sliceTo(&abi.classifySystem(ty, zcu), .none);
8345
8346 for (classes) |class| switch (class) {
8347 .integer => {
8348 const param_int_regs = abi.Registers.Integer.function_arg_regs;
8349 if (param_int_reg_i >= param_int_regs.len) break;
8350
8351 const param_int_reg = param_int_regs[param_int_reg_i];
8352 param_int_reg_i += 1;
8353
8354 arg_mcv[arg_mcv_i] = .{ .register = param_int_reg };
8355 arg_mcv_i += 1;
8356 },
8357 .float => {
8358 const param_float_regs = abi.Registers.Float.function_arg_regs;
8359 if (param_float_reg_i >= param_float_regs.len) break;
8360
8361 const param_float_reg = param_float_regs[param_float_reg_i];
8362 param_float_reg_i += 1;
8363
8364 arg_mcv[arg_mcv_i] = .{ .register = param_float_reg };
8365 arg_mcv_i += 1;
8366 },
8367 .memory => {
8368 const param_int_regs = abi.Registers.Integer.function_arg_regs;
8369
8370 const param_int_reg = param_int_regs[param_int_reg_i];
8371 param_int_reg_i += 1;
8372
8373 arg_mcv[arg_mcv_i] = .{ .indirect = .{ .reg = param_int_reg } };
8374 arg_mcv_i += 1;
8375 },
8376 else => return func.fail("TODO: C calling convention arg class {}", .{class}),
8377 } else {
8378 arg.* = switch (arg_mcv_i) {
8379 else => return func.fail("ty {f} took {} tracking arg indices", .{ ty.fmt(pt), arg_mcv_i }),
8380 1 => arg_mcv[0],
8381 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } },
8382 };
8383 continue;
8384 }
8385
8386 return func.fail("TODO: pass args by stack", .{});
8387 }
8388 },
8389 else => return func.fail("TODO implement function parameters for {} on riscv64", .{cc}),
8390 }
8391
8392 result.stack_byte_count = @intCast(result.stack_align.forward(result.stack_byte_count));
8393 return result;
8394}
8395
8396fn wantSafety(func: *Func) bool {
8397 return switch (func.mod.optimize_mode) {
8398 .Debug => true,
8399 .ReleaseSafe => true,
8400 .ReleaseFast => false,
8401 .ReleaseSmall => false,
8402 };
8403}
8404
8405fn fail(func: *const Func, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
8406 @branchHint(.cold);
8407 const zcu = func.pt.zcu;
8408 switch (func.owner) {
8409 .nav_index => |i| return zcu.codegenFail(i, format, args),
8410 .lazy_sym => |s| return zcu.codegenFailType(s.ty, format, args),
8411 }
8412 return error.CodegenFail;
8413}
8414
8415fn failMsg(func: *const Func, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } {
8416 @branchHint(.cold);
8417 const zcu = func.pt.zcu;
8418 switch (func.owner) {
8419 .nav_index => |i| return zcu.codegenFailMsg(i, msg),
8420 .lazy_sym => |s| return zcu.codegenFailTypeMsg(s.ty, msg),
8421 }
8422 return error.CodegenFail;
8423}
8424
8425fn parseRegName(name: []const u8) ?Register {
8426 // The `fp` alias for `s0` is awkward to fit into the current `Register` scheme, so for now we
8427 // special-case it here.
8428 if (std.mem.eql(u8, name, "fp")) return .s0;
8429
8430 return std.meta.stringToEnum(Register, name);
8431}
8432
8433fn typeOf(func: *Func, inst: Air.Inst.Ref) Type {
8434 return func.air.typeOf(inst, &func.pt.zcu.intern_pool);
8435}
8436
8437fn typeOfIndex(func: *Func, inst: Air.Inst.Index) Type {
8438 const zcu = func.pt.zcu;
8439 return switch (func.air.instructions.items(.tag)[@intFromEnum(inst)]) {
8440 .loop_switch_br => func.typeOf(func.air.unwrapSwitch(inst).operand),
8441 else => func.air.typeOfIndex(inst, &zcu.intern_pool),
8442 };
8443}
8444
8445fn hasFeature(func: *Func, feature: Target.riscv.Feature) bool {
8446 return func.target.cpu.has(.riscv, feature);
8447}
8448
8449pub fn errUnionPayloadOffset(payload_ty: Type, zcu: *Zcu) u64 {
8450 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0;
8451 const payload_align = payload_ty.abiAlignment(zcu);
8452 const error_align = Type.anyerror.abiAlignment(zcu);
8453 if (payload_align.compare(.gte, error_align) or !payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
8454 return 0;
8455 } else {
8456 return payload_align.forward(Type.anyerror.abiSize(zcu));
8457 }
8458}
8459
8460pub fn errUnionErrorOffset(payload_ty: Type, zcu: *Zcu) u64 {
8461 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) return 0;
8462 const payload_align = payload_ty.abiAlignment(zcu);
8463 const error_align = Type.anyerror.abiAlignment(zcu);
8464 if (payload_align.compare(.gte, error_align) and payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
8465 return error_align.forward(payload_ty.abiSize(zcu));
8466 } else {
8467 return 0;
8468 }
8469}
8470
8471fn promoteInt(func: *Func, ty: Type) Type {
8472 const pt = func.pt;
8473 const zcu = pt.zcu;
8474 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
8475 .bool_type => .{ .signedness = .unsigned, .bits = 1 },
8476 else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return ty,
8477 };
8478 for ([_]Type{
8479 Type.c_int, Type.c_uint,
8480 Type.c_long, Type.c_ulong,
8481 Type.c_longlong, Type.c_ulonglong,
8482 }) |promote_ty| {
8483 const promote_info = promote_ty.intInfo(zcu);
8484 if (int_info.signedness == .signed and promote_info.signedness == .unsigned) continue;
8485 if (int_info.bits + @intFromBool(int_info.signedness == .unsigned and
8486 promote_info.signedness == .signed) <= promote_info.bits) return promote_ty;
8487 }
8488 return ty;
8489}
8490
8491fn promoteVarArg(func: *Func, ty: Type) Type {
8492 if (!ty.isRuntimeFloat()) return func.promoteInt(ty);
8493 switch (ty.floatBits(func.target)) {
8494 32, 64 => return Type.f64,
8495 else => |float_bits| {
8496 assert(float_bits == func.target.cTypeBitSize(.longdouble));
8497 return Type.c_longdouble;
8498 },
8499 }
8500}