1pt: Zcu.PerThread,
    2target: *const std.Target,
    3air: Air,
    4nav_index: InternPool.Nav.Index,
    5
    6// Blocks
    7def_order: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, void),
    8blocks: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, Block),
    9loops: std.AutoArrayHashMapUnmanaged(Air.Inst.Index, Loop),
   10active_loops: std.ArrayList(Loop.Index),
   11loop_live: struct {
   12    set: std.AutoArrayHashMapUnmanaged(struct { Loop.Index, Air.Inst.Index }, void),
   13    list: std.ArrayList(Air.Inst.Index),
   14},
   15dom_start: u32,
   16dom_len: u32,
   17dom: std.ArrayList(DomInt),
   18
   19// Wip Mir
   20saved_registers: std.enums.EnumSet(Register.Alias),
   21instructions: std.ArrayList(codegen.aarch64.encoding.Instruction),
   22literals: std.ArrayList(u32),
   23nav_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Nav),
   24uav_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Uav),
   25lazy_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Lazy),
   26global_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Global),
   27literal_relocs: std.ArrayList(codegen.aarch64.Mir.Reloc.Literal),
   28
   29// Stack Frame
   30returns: bool,
   31va_list: union(enum) {
   32    other: Value.Indirect,
   33    sysv: struct {
   34        __stack: Value.Indirect,
   35        __gr_top: Value.Indirect,
   36        __vr_top: Value.Indirect,
   37        __gr_offs: i32,
   38        __vr_offs: i32,
   39    },
   40},
   41stack_size: u24,
   42stack_align: InternPool.Alignment,
   43
   44// Value Tracking
   45live_registers: LiveRegisters,
   46live_values: std.AutoHashMapUnmanaged(Air.Inst.Index, Value.Index),
   47values: std.ArrayList(Value),
   48
   49pub const LiveRegisters = std.enums.EnumArray(Register.Alias, Value.Index);
   50
   51pub const Block = struct {
   52    live_registers: LiveRegisters,
   53    target_label: u32,
   54
   55    pub const main: Air.Inst.Index = @enumFromInt(
   56        std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
   57    );
   58
   59    fn branch(target_block: *const Block, isel: *Select) !void {
   60        if (isel.instructions.items.len > target_block.target_label) {
   61            try isel.emit(.b(@intCast((isel.instructions.items.len + 1 - target_block.target_label) << 2)));
   62        }
   63        try isel.merge(&target_block.live_registers, .{});
   64    }
   65};
   66
   67pub const Loop = struct {
   68    def_order: u32,
   69    dom: u32,
   70    depth: u32,
   71    live: u32,
   72    live_registers: LiveRegisters,
   73    repeat_list: u32,
   74
   75    pub const invalid: Air.Inst.Index = @enumFromInt(
   76        std.math.maxInt(@typeInfo(Air.Inst.Index).@"enum".tag_type),
   77    );
   78
   79    pub const Index = enum(u32) {
   80        _,
   81
   82        fn inst(li: Loop.Index, isel: *Select) Air.Inst.Index {
   83            return isel.loops.keys()[@intFromEnum(li)];
   84        }
   85
   86        fn get(li: Loop.Index, isel: *Select) *Loop {
   87            return &isel.loops.values()[@intFromEnum(li)];
   88        }
   89    };
   90
   91    pub const empty_list: u32 = std.math.maxInt(u32);
   92
   93    fn branch(target_loop: *Loop, isel: *Select) !void {
   94        try isel.instructions.ensureUnusedCapacity(isel.pt.zcu.gpa, 1);
   95        const repeat_list_tail = target_loop.repeat_list;
   96        target_loop.repeat_list = @intCast(isel.instructions.items.len);
   97        isel.instructions.appendAssumeCapacity(@bitCast(repeat_list_tail));
   98        try isel.merge(&target_loop.live_registers, .{});
   99    }
  100};
  101
  102pub fn deinit(isel: *Select) void {
  103    const gpa = isel.pt.zcu.gpa;
  104
  105    isel.def_order.deinit(gpa);
  106    isel.blocks.deinit(gpa);
  107    isel.loops.deinit(gpa);
  108    isel.active_loops.deinit(gpa);
  109    isel.loop_live.set.deinit(gpa);
  110    isel.loop_live.list.deinit(gpa);
  111    isel.dom.deinit(gpa);
  112
  113    isel.instructions.deinit(gpa);
  114    isel.literals.deinit(gpa);
  115    isel.nav_relocs.deinit(gpa);
  116    isel.uav_relocs.deinit(gpa);
  117    isel.lazy_relocs.deinit(gpa);
  118    isel.global_relocs.deinit(gpa);
  119    isel.literal_relocs.deinit(gpa);
  120
  121    isel.live_values.deinit(gpa);
  122    isel.values.deinit(gpa);
  123
  124    isel.* = undefined;
  125}
  126
  127pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
  128    const zcu = isel.pt.zcu;
  129    const ip = &zcu.intern_pool;
  130    const gpa = zcu.gpa;
  131    const air_tags = isel.air.instructions.items(.tag);
  132    const air_data = isel.air.instructions.items(.data);
  133    var air_body_index: usize = 0;
  134    var air_inst_index = air_body[air_body_index];
  135    const initial_def_order_len = isel.def_order.count();
  136    air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {
  137        // No "scalarize" legalizations are enabled, so these instructions never appear.
  138        .legalize_vec_elem_val => unreachable,
  139        .legalize_vec_store_elem => unreachable,
  140        // No soft float legalizations are enabled.
  141        .legalize_compiler_rt_call => unreachable,
  142
  143        .arg,
  144        .ret_addr,
  145        .frame_addr,
  146        .err_return_trace,
  147        .save_err_return_trace_index,
  148        .runtime_nav_ptr,
  149        .c_va_start,
  150        => {
  151            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  152
  153            air_body_index += 1;
  154            air_inst_index = air_body[air_body_index];
  155            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  156        },
  157        .add,
  158        .add_safe,
  159        .add_optimized,
  160        .add_wrap,
  161        .add_sat,
  162        .sub,
  163        .sub_safe,
  164        .sub_optimized,
  165        .sub_wrap,
  166        .sub_sat,
  167        .mul,
  168        .mul_safe,
  169        .mul_optimized,
  170        .mul_wrap,
  171        .mul_sat,
  172        .div_float,
  173        .div_float_optimized,
  174        .div_trunc,
  175        .div_trunc_optimized,
  176        .div_floor,
  177        .div_floor_optimized,
  178        .div_exact,
  179        .div_exact_optimized,
  180        .rem,
  181        .rem_optimized,
  182        .mod,
  183        .mod_optimized,
  184        .max,
  185        .min,
  186        .bit_and,
  187        .bit_or,
  188        .shr,
  189        .shr_exact,
  190        .shl,
  191        .shl_exact,
  192        .shl_sat,
  193        .xor,
  194        .cmp_lt,
  195        .cmp_lt_optimized,
  196        .cmp_lte,
  197        .cmp_lte_optimized,
  198        .cmp_eq,
  199        .cmp_eq_optimized,
  200        .cmp_gte,
  201        .cmp_gte_optimized,
  202        .cmp_gt,
  203        .cmp_gt_optimized,
  204        .cmp_neq,
  205        .cmp_neq_optimized,
  206        .bool_and,
  207        .bool_or,
  208        .array_elem_val,
  209        .slice_elem_val,
  210        .ptr_elem_val,
  211        => {
  212            const bin_op = air_data[@intFromEnum(air_inst_index)].bin_op;
  213
  214            try isel.analyzeUse(bin_op.lhs);
  215            try isel.analyzeUse(bin_op.rhs);
  216            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  217
  218            air_body_index += 1;
  219            air_inst_index = air_body[air_body_index];
  220            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  221        },
  222        .ptr_add,
  223        .ptr_sub,
  224        .add_with_overflow,
  225        .sub_with_overflow,
  226        .mul_with_overflow,
  227        .shl_with_overflow,
  228        .slice,
  229        .slice_elem_ptr,
  230        .ptr_elem_ptr,
  231        => {
  232            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  233            const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
  234
  235            try isel.analyzeUse(bin_op.lhs);
  236            try isel.analyzeUse(bin_op.rhs);
  237            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  238
  239            air_body_index += 1;
  240            air_inst_index = air_body[air_body_index];
  241            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  242        },
  243        .alloc => {
  244            const ty = air_data[@intFromEnum(air_inst_index)].ty;
  245
  246            isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu));
  247            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  248
  249            air_body_index += 1;
  250            air_inst_index = air_body[air_body_index];
  251            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  252        },
  253        .inferred_alloc,
  254        .inferred_alloc_comptime,
  255        .wasm_memory_size,
  256        .wasm_memory_grow,
  257        .work_item_id,
  258        .work_group_size,
  259        .work_group_id,
  260        => unreachable,
  261        .ret_ptr => {
  262            const ty = air_data[@intFromEnum(air_inst_index)].ty;
  263
  264            if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
  265                .unallocated, .stack_slot => isel.stack_align = isel.stack_align.maxStrict(ty.ptrAlignment(zcu)),
  266                .value, .constant => unreachable,
  267                .address => |address_vi| try isel.live_values.putNoClobber(gpa, air_inst_index, address_vi.ref(isel)),
  268            };
  269            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  270
  271            air_body_index += 1;
  272            air_inst_index = air_body[air_body_index];
  273            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  274        },
  275        .assembly => {
  276            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  277            const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
  278            const operands: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0 .. extra.data.flags.outputs_len + extra.data.inputs_len]);
  279
  280            for (operands) |operand| if (operand != .none) try isel.analyzeUse(operand);
  281            if (ty_pl.ty != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  282
  283            air_body_index += 1;
  284            air_inst_index = air_body[air_body_index];
  285            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  286        },
  287        .not,
  288        .clz,
  289        .ctz,
  290        .popcount,
  291        .byte_swap,
  292        .bit_reverse,
  293        .abs,
  294        .load,
  295        .fptrunc,
  296        .fpext,
  297        .intcast,
  298        .intcast_safe,
  299        .trunc,
  300        .optional_payload,
  301        .optional_payload_ptr,
  302        .optional_payload_ptr_set,
  303        .wrap_optional,
  304        .unwrap_errunion_payload,
  305        .unwrap_errunion_err,
  306        .unwrap_errunion_payload_ptr,
  307        .unwrap_errunion_err_ptr,
  308        .errunion_payload_ptr_set,
  309        .wrap_errunion_payload,
  310        .wrap_errunion_err,
  311        .struct_field_ptr_index_0,
  312        .struct_field_ptr_index_1,
  313        .struct_field_ptr_index_2,
  314        .struct_field_ptr_index_3,
  315        .get_union_tag,
  316        .ptr_slice_len_ptr,
  317        .ptr_slice_ptr_ptr,
  318        .array_to_slice,
  319        .int_from_float,
  320        .int_from_float_optimized,
  321        .int_from_float_safe,
  322        .int_from_float_optimized_safe,
  323        .float_from_int,
  324        .splat,
  325        .error_set_has_value,
  326        .addrspace_cast,
  327        .c_va_arg,
  328        .c_va_copy,
  329        => {
  330            const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
  331
  332            try isel.analyzeUse(ty_op.operand);
  333            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  334
  335            air_body_index += 1;
  336            air_inst_index = air_body[air_body_index];
  337            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  338        },
  339        .bitcast => {
  340            const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
  341            maybe_noop: {
  342                if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
  343                if (true) break :maybe_noop;
  344                if (ty_op.operand.toIndex()) |src_air_inst_index| {
  345                    if (isel.hints.get(src_air_inst_index)) |hint_vpsi| {
  346                        try isel.hints.putNoClobber(gpa, air_inst_index, hint_vpsi);
  347                    }
  348                }
  349            }
  350            try isel.analyzeUse(ty_op.operand);
  351            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  352
  353            air_body_index += 1;
  354            air_inst_index = air_body[air_body_index];
  355            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  356        },
  357        inline .block, .dbg_inline_block => |air_tag| {
  358            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  359            const extra = isel.air.extraData(switch (air_tag) {
  360                else => comptime unreachable,
  361                .block => Air.Block,
  362                .dbg_inline_block => Air.DbgInlineBlock,
  363            }, ty_pl.payload);
  364            const result_ty = ty_pl.ty.toInterned().?;
  365
  366            if (result_ty == .noreturn_type) {
  367                try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
  368
  369                air_body_index += 1;
  370                break :air_tag;
  371            }
  372
  373            assert(!(try isel.blocks.getOrPut(gpa, air_inst_index)).found_existing);
  374            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
  375            const block_entry = isel.blocks.pop().?;
  376            assert(block_entry.key == air_inst_index);
  377
  378            if (result_ty != .void_type) try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  379
  380            air_body_index += 1;
  381            air_inst_index = air_body[air_body_index];
  382            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  383        },
  384        .loop => {
  385            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  386            const extra = isel.air.extraData(Air.Block, ty_pl.payload);
  387
  388            const initial_dom_start = isel.dom_start;
  389            const initial_dom_len = isel.dom_len;
  390            isel.dom_start = @intCast(isel.dom.items.len);
  391            isel.dom_len = @intCast(isel.blocks.count());
  392            try isel.active_loops.append(gpa, @enumFromInt(isel.loops.count()));
  393            try isel.loops.putNoClobber(gpa, air_inst_index, .{
  394                .def_order = @intCast(isel.def_order.count()),
  395                .dom = isel.dom_start,
  396                .depth = isel.dom_len,
  397                .live = 0,
  398                .live_registers = undefined,
  399                .repeat_list = undefined,
  400            });
  401            try isel.dom.appendNTimes(gpa, 0, std.math.divCeil(usize, isel.dom_len, @bitSizeOf(DomInt)) catch unreachable);
  402            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
  403            for (
  404                isel.dom.items[initial_dom_start..].ptr,
  405                isel.dom.items[isel.dom_start..][0 .. std.math.divCeil(usize, initial_dom_len, @bitSizeOf(DomInt)) catch unreachable],
  406            ) |*initial_dom, loop_dom| initial_dom.* |= loop_dom;
  407            isel.dom_start = initial_dom_start;
  408            isel.dom_len = initial_dom_len;
  409            assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
  410
  411            air_body_index += 1;
  412        },
  413        .repeat, .trap, .unreach => air_body_index += 1,
  414        .br => {
  415            const br = air_data[@intFromEnum(air_inst_index)].br;
  416            const block_index = isel.blocks.getIndex(br.block_inst).?;
  417            if (block_index < isel.dom_len) isel.dom.items[isel.dom_start + block_index / @bitSizeOf(DomInt)] |= @as(DomInt, 1) << @truncate(block_index);
  418            try isel.analyzeUse(br.operand);
  419
  420            air_body_index += 1;
  421        },
  422        .breakpoint, .dbg_stmt, .dbg_empty_stmt, .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline, .c_va_end => {
  423            air_body_index += 1;
  424            air_inst_index = air_body[air_body_index];
  425            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  426        },
  427        .call,
  428        .call_always_tail,
  429        .call_never_tail,
  430        .call_never_inline,
  431        => {
  432            const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
  433            const extra = isel.air.extraData(Air.Call, pl_op.payload);
  434            const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
  435            isel.saved_registers.insert(.lr);
  436            const callee_ty = isel.air.typeOf(pl_op.operand, ip);
  437            const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
  438                else => unreachable,
  439                .func_type => |func_type| func_type,
  440                .ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
  441            };
  442
  443            try isel.analyzeUse(pl_op.operand);
  444            var param_it: CallAbiIterator = .init;
  445            for (args, 0..) |arg, arg_index| {
  446                const restore_values_len = isel.values.items.len;
  447                defer isel.values.shrinkRetainingCapacity(restore_values_len);
  448                const param_vi = param_vi: {
  449                    const param_ty = isel.air.typeOf(arg, ip);
  450                    if (arg_index >= func_info.param_types.len) {
  451                        assert(func_info.is_var_args);
  452                        switch (isel.va_list) {
  453                            .other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
  454                            .sysv => {},
  455                        }
  456                    }
  457                    break :param_vi try param_it.param(isel, param_ty);
  458                } orelse continue;
  459                defer param_vi.deref(isel);
  460                const passed_vi = switch (param_vi.parent(isel)) {
  461                    .unallocated, .stack_slot => param_vi,
  462                    .value, .constant => unreachable,
  463                    .address => |address_vi| address_vi,
  464                };
  465                switch (passed_vi.parent(isel)) {
  466                    .unallocated => {},
  467                    .stack_slot => |stack_slot| {
  468                        assert(stack_slot.base == .sp);
  469                        isel.stack_size = @max(
  470                            isel.stack_size,
  471                            stack_slot.offset + @as(u24, @intCast(passed_vi.size(isel))),
  472                        );
  473                    },
  474                    .value, .constant, .address => unreachable,
  475                }
  476
  477                try isel.analyzeUse(arg);
  478            }
  479
  480            var ret_it: CallAbiIterator = .init;
  481            if (try ret_it.ret(isel, isel.air.typeOfIndex(air_inst_index, ip))) |ret_vi| {
  482                tracking_log.debug("${d} <- %{d}", .{ @intFromEnum(ret_vi), @intFromEnum(air_inst_index) });
  483                switch (ret_vi.parent(isel)) {
  484                    .unallocated, .stack_slot => {},
  485                    .value, .constant => unreachable,
  486                    .address => |address_vi| {
  487                        defer address_vi.deref(isel);
  488                        const ret_value = ret_vi.get(isel);
  489                        ret_value.flags.parent_tag = .unallocated;
  490                        ret_value.parent_payload = .{ .unallocated = {} };
  491                    },
  492                }
  493                try isel.live_values.putNoClobber(gpa, air_inst_index, ret_vi);
  494
  495                try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  496            }
  497
  498            air_body_index += 1;
  499            air_inst_index = air_body[air_body_index];
  500            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  501        },
  502        .sqrt,
  503        .sin,
  504        .cos,
  505        .tan,
  506        .exp,
  507        .exp2,
  508        .log,
  509        .log2,
  510        .log10,
  511        .floor,
  512        .ceil,
  513        .round,
  514        .trunc_float,
  515        .neg,
  516        .neg_optimized,
  517        .is_null,
  518        .is_non_null,
  519        .is_null_ptr,
  520        .is_non_null_ptr,
  521        .is_err,
  522        .is_non_err,
  523        .is_err_ptr,
  524        .is_non_err_ptr,
  525        .is_named_enum_value,
  526        .tag_name,
  527        .error_name,
  528        .cmp_lt_errors_len,
  529        => {
  530            const un_op = air_data[@intFromEnum(air_inst_index)].un_op;
  531
  532            try isel.analyzeUse(un_op);
  533            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  534
  535            air_body_index += 1;
  536            air_inst_index = air_body[air_body_index];
  537            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  538        },
  539        .cmp_vector, .cmp_vector_optimized => {
  540            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  541            const extra = isel.air.extraData(Air.VectorCmp, ty_pl.payload).data;
  542
  543            try isel.analyzeUse(extra.lhs);
  544            try isel.analyzeUse(extra.rhs);
  545            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  546
  547            air_body_index += 1;
  548            air_inst_index = air_body[air_body_index];
  549            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  550        },
  551        .cond_br => {
  552            const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
  553            const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
  554
  555            try isel.analyzeUse(pl_op.operand);
  556
  557            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
  558            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
  559
  560            air_body_index += 1;
  561        },
  562        .switch_br => {
  563            const switch_br = isel.air.unwrapSwitch(air_inst_index);
  564
  565            try isel.analyzeUse(switch_br.operand);
  566
  567            var cases_it = switch_br.iterateCases();
  568            while (cases_it.next()) |case| try isel.analyze(case.body);
  569            if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
  570
  571            air_body_index += 1;
  572        },
  573        .loop_switch_br => {
  574            const switch_br = isel.air.unwrapSwitch(air_inst_index);
  575
  576            const initial_dom_start = isel.dom_start;
  577            const initial_dom_len = isel.dom_len;
  578            isel.dom_start = @intCast(isel.dom.items.len);
  579            isel.dom_len = @intCast(isel.blocks.count());
  580            try isel.active_loops.append(gpa, @enumFromInt(isel.loops.count()));
  581            try isel.loops.putNoClobber(gpa, air_inst_index, .{
  582                .def_order = @intCast(isel.def_order.count()),
  583                .dom = isel.dom_start,
  584                .depth = isel.dom_len,
  585                .live = 0,
  586                .live_registers = undefined,
  587                .repeat_list = undefined,
  588            });
  589            try isel.dom.appendNTimes(gpa, 0, std.math.divCeil(usize, isel.dom_len, @bitSizeOf(DomInt)) catch unreachable);
  590
  591            var cases_it = switch_br.iterateCases();
  592            while (cases_it.next()) |case| try isel.analyze(case.body);
  593            if (switch_br.else_body_len > 0) try isel.analyze(cases_it.elseBody());
  594
  595            for (
  596                isel.dom.items[initial_dom_start..].ptr,
  597                isel.dom.items[isel.dom_start..][0 .. std.math.divCeil(usize, initial_dom_len, @bitSizeOf(DomInt)) catch unreachable],
  598            ) |*initial_dom, loop_dom| initial_dom.* |= loop_dom;
  599            isel.dom_start = initial_dom_start;
  600            isel.dom_len = initial_dom_len;
  601            assert(isel.active_loops.pop().?.inst(isel) == air_inst_index);
  602
  603            air_body_index += 1;
  604        },
  605        .switch_dispatch => {
  606            const br = air_data[@intFromEnum(air_inst_index)].br;
  607
  608            try isel.analyzeUse(br.operand);
  609
  610            air_body_index += 1;
  611        },
  612        .@"try", .try_cold => {
  613            const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
  614            const extra = isel.air.extraData(Air.Try, pl_op.payload);
  615
  616            try isel.analyzeUse(pl_op.operand);
  617            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
  618            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  619
  620            air_body_index += 1;
  621            air_inst_index = air_body[air_body_index];
  622            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  623        },
  624        .try_ptr, .try_ptr_cold => {
  625            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  626            const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload);
  627
  628            try isel.analyzeUse(extra.data.ptr);
  629            try isel.analyze(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
  630            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  631
  632            air_body_index += 1;
  633            air_inst_index = air_body[air_body_index];
  634            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  635        },
  636        .ret, .ret_safe, .ret_load => {
  637            const un_op = air_data[@intFromEnum(air_inst_index)].un_op;
  638            isel.returns = true;
  639
  640            const block_index = 0;
  641            assert(isel.blocks.keys()[block_index] == Block.main);
  642            if (isel.dom_len > 0) isel.dom.items[isel.dom_start] |= 1 << block_index;
  643
  644            try isel.analyzeUse(un_op);
  645
  646            air_body_index += 1;
  647        },
  648        .store,
  649        .store_safe,
  650        .set_union_tag,
  651        .memset,
  652        .memset_safe,
  653        .memcpy,
  654        .memmove,
  655        .atomic_store_unordered,
  656        .atomic_store_monotonic,
  657        .atomic_store_release,
  658        .atomic_store_seq_cst,
  659        => {
  660            const bin_op = air_data[@intFromEnum(air_inst_index)].bin_op;
  661
  662            try isel.analyzeUse(bin_op.lhs);
  663            try isel.analyzeUse(bin_op.rhs);
  664
  665            air_body_index += 1;
  666            air_inst_index = air_body[air_body_index];
  667            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  668        },
  669        .struct_field_ptr, .struct_field_val => {
  670            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  671            const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
  672
  673            try isel.analyzeUse(extra.struct_operand);
  674            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  675
  676            air_body_index += 1;
  677            air_inst_index = air_body[air_body_index];
  678            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  679        },
  680        .slice_len => {
  681            const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
  682
  683            try isel.analyzeUse(ty_op.operand);
  684            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  685
  686            const slice_vi = try isel.use(ty_op.operand);
  687            var len_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 8, 8);
  688            if (try len_part_it.only(isel)) |len_part_vi|
  689                try isel.live_values.putNoClobber(gpa, air_inst_index, len_part_vi.ref(isel));
  690
  691            air_body_index += 1;
  692            air_inst_index = air_body[air_body_index];
  693            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  694        },
  695        .slice_ptr => {
  696            const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
  697
  698            try isel.analyzeUse(ty_op.operand);
  699            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  700
  701            const slice_vi = try isel.use(ty_op.operand);
  702            var ptr_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 0, 8);
  703            if (try ptr_part_it.only(isel)) |ptr_part_vi|
  704                try isel.live_values.putNoClobber(gpa, air_inst_index, ptr_part_vi.ref(isel));
  705
  706            air_body_index += 1;
  707            air_inst_index = air_body[air_body_index];
  708            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  709        },
  710        .reduce, .reduce_optimized => {
  711            const reduce = air_data[@intFromEnum(air_inst_index)].reduce;
  712
  713            try isel.analyzeUse(reduce.operand);
  714            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  715
  716            air_body_index += 1;
  717            air_inst_index = air_body[air_body_index];
  718            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  719        },
  720        .shuffle_one => {
  721            const extra = isel.air.unwrapShuffleOne(zcu, air_inst_index);
  722
  723            try isel.analyzeUse(extra.operand);
  724            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  725
  726            air_body_index += 1;
  727            air_inst_index = air_body[air_body_index];
  728            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  729        },
  730        .shuffle_two => {
  731            const extra = isel.air.unwrapShuffleTwo(zcu, air_inst_index);
  732
  733            try isel.analyzeUse(extra.operand_a);
  734            try isel.analyzeUse(extra.operand_b);
  735            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  736
  737            air_body_index += 1;
  738            air_inst_index = air_body[air_body_index];
  739            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  740        },
  741        .select, .mul_add => {
  742            const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
  743            const bin_op = isel.air.extraData(Air.Bin, pl_op.payload).data;
  744
  745            try isel.analyzeUse(pl_op.operand);
  746            try isel.analyzeUse(bin_op.lhs);
  747            try isel.analyzeUse(bin_op.rhs);
  748            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  749
  750            air_body_index += 1;
  751            air_inst_index = air_body[air_body_index];
  752            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  753        },
  754        .cmpxchg_weak, .cmpxchg_strong => {
  755            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  756            const extra = isel.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
  757
  758            try isel.analyzeUse(extra.ptr);
  759            try isel.analyzeUse(extra.expected_value);
  760            try isel.analyzeUse(extra.new_value);
  761            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  762
  763            air_body_index += 1;
  764            air_inst_index = air_body[air_body_index];
  765            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  766        },
  767        .atomic_load => {
  768            const atomic_load = air_data[@intFromEnum(air_inst_index)].atomic_load;
  769
  770            try isel.analyzeUse(atomic_load.ptr);
  771            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  772
  773            air_body_index += 1;
  774            air_inst_index = air_body[air_body_index];
  775            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  776        },
  777        .atomic_rmw => {
  778            const pl_op = air_data[@intFromEnum(air_inst_index)].pl_op;
  779            const extra = isel.air.extraData(Air.AtomicRmw, pl_op.payload).data;
  780
  781            try isel.analyzeUse(extra.operand);
  782            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  783
  784            air_body_index += 1;
  785            air_inst_index = air_body[air_body_index];
  786            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  787        },
  788        .aggregate_init => {
  789            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  790            const elements: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(ty_pl.ty.toType().arrayLen(zcu))]);
  791
  792            for (elements) |element| try isel.analyzeUse(element);
  793            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  794
  795            air_body_index += 1;
  796            air_inst_index = air_body[air_body_index];
  797            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  798        },
  799        .union_init => {
  800            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  801            const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
  802
  803            try isel.analyzeUse(extra.init);
  804            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  805
  806            air_body_index += 1;
  807            air_inst_index = air_body[air_body_index];
  808            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  809        },
  810        .prefetch => {
  811            const prefetch = air_data[@intFromEnum(air_inst_index)].prefetch;
  812
  813            try isel.analyzeUse(prefetch.ptr);
  814
  815            air_body_index += 1;
  816            air_inst_index = air_body[air_body_index];
  817            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  818        },
  819        .field_parent_ptr => {
  820            const ty_pl = air_data[@intFromEnum(air_inst_index)].ty_pl;
  821            const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
  822
  823            try isel.analyzeUse(extra.field_ptr);
  824            try isel.def_order.putNoClobber(gpa, air_inst_index, {});
  825
  826            air_body_index += 1;
  827            air_inst_index = air_body[air_body_index];
  828            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  829        },
  830        .set_err_return_trace => {
  831            const un_op = air_data[@intFromEnum(air_inst_index)].un_op;
  832
  833            try isel.analyzeUse(un_op);
  834
  835            air_body_index += 1;
  836            air_inst_index = air_body[air_body_index];
  837            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
  838        },
  839    }
  840    assert(air_body_index == air_body.len);
  841    isel.def_order.shrinkRetainingCapacity(initial_def_order_len);
  842}
  843
  844fn analyzeUse(isel: *Select, air_ref: Air.Inst.Ref) !void {
  845    const air_inst_index = air_ref.toIndex() orelse return;
  846    const def_order_index = isel.def_order.getIndex(air_inst_index).?;
  847
  848    // Loop liveness
  849    var active_loop_index = isel.active_loops.items.len;
  850    while (active_loop_index > 0) {
  851        const prev_active_loop_index = active_loop_index - 1;
  852        const active_loop = isel.active_loops.items[prev_active_loop_index];
  853        if (def_order_index >= active_loop.get(isel).def_order) break;
  854        active_loop_index = prev_active_loop_index;
  855    }
  856    if (active_loop_index < isel.active_loops.items.len) {
  857        const active_loop = isel.active_loops.items[active_loop_index];
  858        const loop_live_gop =
  859            try isel.loop_live.set.getOrPut(isel.pt.zcu.gpa, .{ active_loop, air_inst_index });
  860        if (!loop_live_gop.found_existing) active_loop.get(isel).live += 1;
  861    }
  862}
  863
  864pub fn finishAnalysis(isel: *Select) !void {
  865    const gpa = isel.pt.zcu.gpa;
  866
  867    // Loop Liveness
  868    if (isel.loops.count() > 0) {
  869        try isel.loops.ensureUnusedCapacity(gpa, 1);
  870
  871        const loop_live_len: u32 = @intCast(isel.loop_live.set.count());
  872        if (loop_live_len > 0) {
  873            try isel.loop_live.list.resize(gpa, loop_live_len);
  874
  875            const loops = isel.loops.values();
  876            for (loops[1..], loops[0 .. loops.len - 1]) |*loop, prev_loop| loop.live += prev_loop.live;
  877            assert(loops[loops.len - 1].live == loop_live_len);
  878
  879            for (isel.loop_live.set.keys()) |entry| {
  880                const loop, const inst = entry;
  881                const loop_live = &loop.get(isel).live;
  882                loop_live.* -= 1;
  883                isel.loop_live.list.items[loop_live.*] = inst;
  884            }
  885            assert(loops[0].live == 0);
  886        }
  887
  888        const invalid_gop = isel.loops.getOrPutAssumeCapacity(Loop.invalid);
  889        assert(!invalid_gop.found_existing);
  890        invalid_gop.value_ptr.live = loop_live_len;
  891    }
  892}
  893
  894pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, CodegenFail }!void {
  895    const zcu = isel.pt.zcu;
  896    const ip = &zcu.intern_pool;
  897    const gpa = zcu.gpa;
  898
  899    {
  900        var live_reg_it = isel.live_registers.iterator();
  901        while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
  902            _ => {
  903                const ra = &live_reg_entry.value.get(isel).location_payload.small.register;
  904                assert(ra.* == live_reg_entry.key);
  905                ra.* = .zr;
  906                live_reg_entry.value.* = .free;
  907            },
  908            .allocating => live_reg_entry.value.* = .free,
  909            .free => {},
  910        };
  911    }
  912
  913    var air: struct {
  914        isel: *Select,
  915        tag_items: []const Air.Inst.Tag,
  916        data_items: []const Air.Inst.Data,
  917        body: []const Air.Inst.Index,
  918        body_index: u32,
  919        inst_index: Air.Inst.Index,
  920
  921        fn tag(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Tag {
  922            return it.tag_items[@intFromEnum(inst_index)];
  923        }
  924
  925        fn data(it: *@This(), inst_index: Air.Inst.Index) Air.Inst.Data {
  926            return it.data_items[@intFromEnum(inst_index)];
  927        }
  928
  929        fn next(it: *@This()) ?Air.Inst.Tag {
  930            if (it.body_index == 0) {
  931                @branchHint(.unlikely);
  932                return null;
  933            }
  934            it.body_index -= 1;
  935            it.inst_index = it.body[it.body_index];
  936            wip_mir_log.debug("{f}", .{it.fmtAir(it.inst_index)});
  937            return it.tag(it.inst_index);
  938        }
  939
  940        fn fmtAir(it: @This(), inst: Air.Inst.Index) struct {
  941            isel: *Select,
  942            inst: Air.Inst.Index,
  943            pub fn format(fmt_air: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
  944                fmt_air.isel.air.writeInst(writer, fmt_air.inst, fmt_air.isel.pt, null);
  945            }
  946        } {
  947            return .{ .isel = it.isel, .inst = inst };
  948        }
  949    } = .{
  950        .isel = isel,
  951        .tag_items = isel.air.instructions.items(.tag),
  952        .data_items = isel.air.instructions.items(.data),
  953        .body = air_body,
  954        .body_index = @intCast(air_body.len),
  955        .inst_index = undefined,
  956    };
  957    air_tag: switch (air.next().?) {
  958        else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),
  959
  960        // No "scalarize" legalizations are enabled, so these instructions never appear.
  961        .legalize_vec_elem_val => unreachable,
  962        .legalize_vec_store_elem => unreachable,
  963
  964        .arg => {
  965            const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
  966            defer arg_vi.deref(isel);
  967            switch (arg_vi.parent(isel)) {
  968                .unallocated, .stack_slot => if (arg_vi.hint(isel)) |arg_ra| {
  969                    try arg_vi.defLiveIn(isel, arg_ra, comptime &.initFill(.free));
  970                } else {
  971                    var arg_part_it = arg_vi.parts(isel);
  972                    while (arg_part_it.next()) |arg_part| {
  973                        try arg_part.defLiveIn(isel, arg_part.hint(isel).?, comptime &.initFill(.free));
  974                    }
  975                },
  976                .value, .constant => unreachable,
  977                .address => |address_vi| try address_vi.defLiveIn(isel, address_vi.hint(isel).?, comptime &.initFill(.free)),
  978            }
  979            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
  980        },
  981        .add, .add_safe, .add_optimized, .add_wrap, .sub, .sub_safe, .sub_optimized, .sub_wrap => |air_tag| {
  982            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
  983                defer res_vi.value.deref(isel);
  984
  985                const bin_op = air.data(air.inst_index).bin_op;
  986                const ty = isel.air.typeOf(bin_op.lhs, ip);
  987                if (!ty.isRuntimeFloat()) try res_vi.value.addOrSubtract(isel, ty, try isel.use(bin_op.lhs), switch (air_tag) {
  988                    else => unreachable,
  989                    .add, .add_safe, .add_wrap => .add,
  990                    .sub, .sub_safe, .sub_wrap => .sub,
  991                }, try isel.use(bin_op.rhs), .{
  992                    .overflow = switch (air_tag) {
  993                        else => unreachable,
  994                        .add, .sub => .@"unreachable",
  995                        .add_safe, .sub_safe => .{ .panic = .integer_overflow },
  996                        .add_wrap, .sub_wrap => .wrap,
  997                    },
  998                }) else switch (ty.floatBits(isel.target)) {
  999                    else => unreachable,
 1000                    16, 32, 64 => |bits| {
 1001                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1002                        const need_fcvt = switch (bits) {
 1003                            else => unreachable,
 1004                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 1005                            32, 64 => false,
 1006                        };
 1007                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 1008                        const lhs_vi = try isel.use(bin_op.lhs);
 1009                        const rhs_vi = try isel.use(bin_op.rhs);
 1010                        const lhs_mat = try lhs_vi.matReg(isel);
 1011                        const rhs_mat = try rhs_vi.matReg(isel);
 1012                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 1013                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 1014                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 1015                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 1016                        try isel.emit(bits: switch (bits) {
 1017                            else => unreachable,
 1018                            16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
 1019                                else => unreachable,
 1020                                .add, .add_optimized => .fadd(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 1021                                .sub, .sub_optimized => .fsub(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 1022                            },
 1023                            32 => switch (air_tag) {
 1024                                else => unreachable,
 1025                                .add, .add_optimized => .fadd(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 1026                                .sub, .sub_optimized => .fsub(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 1027                            },
 1028                            64 => switch (air_tag) {
 1029                                else => unreachable,
 1030                                .add, .add_optimized => .fadd(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 1031                                .sub, .sub_optimized => .fsub(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 1032                            },
 1033                        });
 1034                        if (need_fcvt) {
 1035                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 1036                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 1037                        }
 1038                        try rhs_mat.finish(isel);
 1039                        try lhs_mat.finish(isel);
 1040                    },
 1041                    80, 128 => |bits| {
 1042                        try call.prepareReturn(isel);
 1043                        switch (bits) {
 1044                            else => unreachable,
 1045                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 1046                            80 => {
 1047                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 1048                                const res_hi16_vi = try res_hi16_it.only(isel);
 1049                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 1050                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 1051                                const res_lo64_vi = try res_lo64_it.only(isel);
 1052                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 1053                            },
 1054                        }
 1055                        try call.finishReturn(isel);
 1056
 1057                        try call.prepareCallee(isel);
 1058                        try isel.global_relocs.append(gpa, .{
 1059                            .name = switch (air_tag) {
 1060                                else => unreachable,
 1061                                .add, .add_optimized => switch (bits) {
 1062                                    else => unreachable,
 1063                                    16 => "__addhf3",
 1064                                    32 => "__addsf3",
 1065                                    64 => "__adddf3",
 1066                                    80 => "__addxf3",
 1067                                    128 => "__addtf3",
 1068                                },
 1069                                .sub, .sub_optimized => switch (bits) {
 1070                                    else => unreachable,
 1071                                    16 => "__subhf3",
 1072                                    32 => "__subsf3",
 1073                                    64 => "__subdf3",
 1074                                    80 => "__subxf3",
 1075                                    128 => "__subtf3",
 1076                                },
 1077                            },
 1078                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 1079                        });
 1080                        try isel.emit(.bl(0));
 1081                        try call.finishCallee(isel);
 1082
 1083                        try call.prepareParams(isel);
 1084                        const lhs_vi = try isel.use(bin_op.lhs);
 1085                        const rhs_vi = try isel.use(bin_op.rhs);
 1086                        switch (bits) {
 1087                            else => unreachable,
 1088                            16, 32, 64, 128 => {
 1089                                try call.paramLiveOut(isel, rhs_vi, .v1);
 1090                                try call.paramLiveOut(isel, lhs_vi, .v0);
 1091                            },
 1092                            80 => {
 1093                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 1094                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 1095                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 1096                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 1097                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 1098                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 1099                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 1100                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 1101                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 1102                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 1103                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 1104                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 1105                            },
 1106                        }
 1107                        try call.finishParams(isel);
 1108                    },
 1109                }
 1110            }
 1111            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1112        },
 1113        .add_sat, .sub_sat => |air_tag| {
 1114            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1115                defer res_vi.value.deref(isel);
 1116
 1117                const bin_op = air.data(air.inst_index).bin_op;
 1118                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1119                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 1120                const int_info = ty.intInfo(zcu);
 1121                switch (int_info.bits) {
 1122                    0 => unreachable,
 1123                    32, 64 => |bits| switch (int_info.signedness) {
 1124                        .signed => return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1125                        .unsigned => {
 1126                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1127                            const lhs_vi = try isel.use(bin_op.lhs);
 1128                            const rhs_vi = try isel.use(bin_op.rhs);
 1129                            const lhs_mat = try lhs_vi.matReg(isel);
 1130                            const rhs_mat = try rhs_vi.matReg(isel);
 1131                            const unsat_res_ra = try isel.allocIntReg();
 1132                            defer isel.freeReg(unsat_res_ra);
 1133                            switch (air_tag) {
 1134                                else => unreachable,
 1135                                .add_sat => switch (bits) {
 1136                                    else => unreachable,
 1137                                    32 => {
 1138                                        try isel.emit(.csinv(res_ra.w(), unsat_res_ra.w(), .wzr, .invert(.cs)));
 1139                                        try isel.emit(.adds(unsat_res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1140                                    },
 1141                                    64 => {
 1142                                        try isel.emit(.csinv(res_ra.x(), unsat_res_ra.x(), .xzr, .invert(.cs)));
 1143                                        try isel.emit(.adds(unsat_res_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
 1144                                    },
 1145                                },
 1146                                .sub_sat => switch (bits) {
 1147                                    else => unreachable,
 1148                                    32 => {
 1149                                        try isel.emit(.csel(res_ra.w(), unsat_res_ra.w(), .wzr, .invert(.cc)));
 1150                                        try isel.emit(.subs(unsat_res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1151                                    },
 1152                                    64 => {
 1153                                        try isel.emit(.csel(res_ra.x(), unsat_res_ra.x(), .xzr, .invert(.cc)));
 1154                                        try isel.emit(.subs(unsat_res_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
 1155                                    },
 1156                                },
 1157                            }
 1158                            try rhs_mat.finish(isel);
 1159                            try lhs_mat.finish(isel);
 1160                        },
 1161                    },
 1162                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1163                }
 1164            }
 1165            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1166        },
 1167        .mul, .mul_optimized, .mul_wrap => |air_tag| {
 1168            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1169                defer res_vi.value.deref(isel);
 1170
 1171                const bin_op = air.data(air.inst_index).bin_op;
 1172                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1173                if (!ty.isRuntimeFloat()) {
 1174                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 1175                    const int_info = ty.intInfo(zcu);
 1176                    switch (int_info.bits) {
 1177                        0 => unreachable,
 1178                        1 => {
 1179                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1180                            switch (int_info.signedness) {
 1181                                .signed => switch (air_tag) {
 1182                                    else => unreachable,
 1183                                    .mul => break :unused try isel.emit(.orr(res_ra.w(), .wzr, .{ .register = .wzr })),
 1184                                    .mul_wrap => {},
 1185                                },
 1186                                .unsigned => {},
 1187                            }
 1188                            const lhs_vi = try isel.use(bin_op.lhs);
 1189                            const rhs_vi = try isel.use(bin_op.rhs);
 1190                            const lhs_mat = try lhs_vi.matReg(isel);
 1191                            const rhs_mat = try rhs_vi.matReg(isel);
 1192                            try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1193                            try rhs_mat.finish(isel);
 1194                            try lhs_mat.finish(isel);
 1195                        },
 1196                        2...32 => |bits| {
 1197                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1198                            switch (air_tag) {
 1199                                else => unreachable,
 1200                                .mul => {},
 1201                                .mul_wrap => switch (bits) {
 1202                                    else => unreachable,
 1203                                    1...31 => try isel.emit(switch (int_info.signedness) {
 1204                                        .signed => .sbfm(res_ra.w(), res_ra.w(), .{
 1205                                            .N = .word,
 1206                                            .immr = 0,
 1207                                            .imms = @intCast(bits - 1),
 1208                                        }),
 1209                                        .unsigned => .ubfm(res_ra.w(), res_ra.w(), .{
 1210                                            .N = .word,
 1211                                            .immr = 0,
 1212                                            .imms = @intCast(bits - 1),
 1213                                        }),
 1214                                    }),
 1215                                    32 => {},
 1216                                },
 1217                            }
 1218                            const lhs_vi = try isel.use(bin_op.lhs);
 1219                            const rhs_vi = try isel.use(bin_op.rhs);
 1220                            const lhs_mat = try lhs_vi.matReg(isel);
 1221                            const rhs_mat = try rhs_vi.matReg(isel);
 1222                            try isel.emit(.madd(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr));
 1223                            try rhs_mat.finish(isel);
 1224                            try lhs_mat.finish(isel);
 1225                        },
 1226                        33...64 => |bits| {
 1227                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1228                            switch (air_tag) {
 1229                                else => unreachable,
 1230                                .mul => {},
 1231                                .mul_wrap => switch (bits) {
 1232                                    else => unreachable,
 1233                                    33...63 => try isel.emit(switch (int_info.signedness) {
 1234                                        .signed => .sbfm(res_ra.x(), res_ra.x(), .{
 1235                                            .N = .doubleword,
 1236                                            .immr = 0,
 1237                                            .imms = @intCast(bits - 1),
 1238                                        }),
 1239                                        .unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
 1240                                            .N = .doubleword,
 1241                                            .immr = 0,
 1242                                            .imms = @intCast(bits - 1),
 1243                                        }),
 1244                                    }),
 1245                                    64 => {},
 1246                                },
 1247                            }
 1248                            const lhs_vi = try isel.use(bin_op.lhs);
 1249                            const rhs_vi = try isel.use(bin_op.rhs);
 1250                            const lhs_mat = try lhs_vi.matReg(isel);
 1251                            const rhs_mat = try rhs_vi.matReg(isel);
 1252                            try isel.emit(.madd(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
 1253                            try rhs_mat.finish(isel);
 1254                            try lhs_mat.finish(isel);
 1255                        },
 1256                        65...128 => |bits| {
 1257                            var res_hi64_it = res_vi.value.field(ty, 8, 8);
 1258                            const res_hi64_vi = try res_hi64_it.only(isel);
 1259                            const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
 1260                            var res_lo64_it = res_vi.value.field(ty, 0, 8);
 1261                            const res_lo64_vi = try res_lo64_it.only(isel);
 1262                            const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
 1263                            if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
 1264                            if (res_hi64_ra) |res_ra| switch (air_tag) {
 1265                                else => unreachable,
 1266                                .mul => {},
 1267                                .mul_wrap => switch (bits) {
 1268                                    else => unreachable,
 1269                                    65...127 => try isel.emit(switch (int_info.signedness) {
 1270                                        .signed => .sbfm(res_ra.x(), res_ra.x(), .{
 1271                                            .N = .doubleword,
 1272                                            .immr = 0,
 1273                                            .imms = @intCast(bits - 1),
 1274                                        }),
 1275                                        .unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
 1276                                            .N = .doubleword,
 1277                                            .immr = 0,
 1278                                            .imms = @intCast(bits - 1),
 1279                                        }),
 1280                                    }),
 1281                                    128 => {},
 1282                                },
 1283                            };
 1284                            const lhs_vi = try isel.use(bin_op.lhs);
 1285                            const rhs_vi = try isel.use(bin_op.rhs);
 1286                            const lhs_lo64_mat, const rhs_lo64_mat = lo64_mat: {
 1287                                const res_hi64_lock: RegLock = if (res_hi64_ra != null and res_lo64_ra != null)
 1288                                    isel.lockReg(res_hi64_ra.?)
 1289                                else
 1290                                    .empty;
 1291                                defer res_hi64_lock.unlock(isel);
 1292
 1293                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 1294                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 1295                                const rhs_lo64_mat = try rhs_lo64_vi.?.matReg(isel);
 1296                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 1297                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 1298                                const lhs_lo64_mat = try lhs_lo64_vi.?.matReg(isel);
 1299                                break :lo64_mat .{ lhs_lo64_mat, rhs_lo64_mat };
 1300                            };
 1301                            if (res_lo64_ra) |res_ra| try isel.emit(.madd(res_ra.x(), lhs_lo64_mat.ra.x(), rhs_lo64_mat.ra.x(), .xzr));
 1302                            if (res_hi64_ra) |res_ra| {
 1303                                var rhs_hi64_it = rhs_vi.field(ty, 8, 8);
 1304                                const rhs_hi64_vi = try rhs_hi64_it.only(isel);
 1305                                const rhs_hi64_mat = try rhs_hi64_vi.?.matReg(isel);
 1306                                var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
 1307                                const lhs_hi64_vi = try lhs_hi64_it.only(isel);
 1308                                const lhs_hi64_mat = try lhs_hi64_vi.?.matReg(isel);
 1309                                const acc_ra = try isel.allocIntReg();
 1310                                defer isel.freeReg(acc_ra);
 1311                                try isel.emit(.madd(res_ra.x(), lhs_hi64_mat.ra.x(), rhs_lo64_mat.ra.x(), acc_ra.x()));
 1312                                try isel.emit(.madd(acc_ra.x(), lhs_lo64_mat.ra.x(), rhs_hi64_mat.ra.x(), acc_ra.x()));
 1313                                try isel.emit(.umulh(acc_ra.x(), lhs_lo64_mat.ra.x(), rhs_lo64_mat.ra.x()));
 1314                                try rhs_hi64_mat.finish(isel);
 1315                                try lhs_hi64_mat.finish(isel);
 1316                            }
 1317                            try rhs_lo64_mat.finish(isel);
 1318                            try lhs_lo64_mat.finish(isel);
 1319                        },
 1320                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1321                    }
 1322                } else switch (ty.floatBits(isel.target)) {
 1323                    else => unreachable,
 1324                    16, 32, 64 => |bits| {
 1325                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1326                        const need_fcvt = switch (bits) {
 1327                            else => unreachable,
 1328                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 1329                            32, 64 => false,
 1330                        };
 1331                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 1332                        const lhs_vi = try isel.use(bin_op.lhs);
 1333                        const rhs_vi = try isel.use(bin_op.rhs);
 1334                        const lhs_mat = try lhs_vi.matReg(isel);
 1335                        const rhs_mat = try rhs_vi.matReg(isel);
 1336                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 1337                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 1338                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 1339                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 1340                        try isel.emit(bits: switch (bits) {
 1341                            else => unreachable,
 1342                            16 => if (need_fcvt)
 1343                                continue :bits 32
 1344                            else
 1345                                .fmul(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 1346                            32 => .fmul(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 1347                            64 => .fmul(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 1348                        });
 1349                        if (need_fcvt) {
 1350                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 1351                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 1352                        }
 1353                        try rhs_mat.finish(isel);
 1354                        try lhs_mat.finish(isel);
 1355                    },
 1356                    80, 128 => |bits| {
 1357                        try call.prepareReturn(isel);
 1358                        switch (bits) {
 1359                            else => unreachable,
 1360                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 1361                            80 => {
 1362                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 1363                                const res_hi16_vi = try res_hi16_it.only(isel);
 1364                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 1365                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 1366                                const res_lo64_vi = try res_lo64_it.only(isel);
 1367                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 1368                            },
 1369                        }
 1370                        try call.finishReturn(isel);
 1371
 1372                        try call.prepareCallee(isel);
 1373                        try isel.global_relocs.append(gpa, .{
 1374                            .name = switch (bits) {
 1375                                else => unreachable,
 1376                                16 => "__mulhf3",
 1377                                32 => "__mulsf3",
 1378                                64 => "__muldf3",
 1379                                80 => "__mulxf3",
 1380                                128 => "__multf3",
 1381                            },
 1382                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 1383                        });
 1384                        try isel.emit(.bl(0));
 1385                        try call.finishCallee(isel);
 1386
 1387                        try call.prepareParams(isel);
 1388                        const lhs_vi = try isel.use(bin_op.lhs);
 1389                        const rhs_vi = try isel.use(bin_op.rhs);
 1390                        switch (bits) {
 1391                            else => unreachable,
 1392                            16, 32, 64, 128 => {
 1393                                try call.paramLiveOut(isel, rhs_vi, .v1);
 1394                                try call.paramLiveOut(isel, lhs_vi, .v0);
 1395                            },
 1396                            80 => {
 1397                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 1398                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 1399                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 1400                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 1401                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 1402                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 1403                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 1404                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 1405                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 1406                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 1407                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 1408                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 1409                            },
 1410                        }
 1411                        try call.finishParams(isel);
 1412                    },
 1413                }
 1414            }
 1415            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1416        },
 1417        .mul_safe => |air_tag| {
 1418            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1419                defer res_vi.value.deref(isel);
 1420
 1421                const bin_op = air.data(air.inst_index).bin_op;
 1422                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1423                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 1424                const int_info = ty.intInfo(zcu);
 1425                switch (int_info.signedness) {
 1426                    .signed => switch (int_info.bits) {
 1427                        0 => unreachable,
 1428                        1 => {
 1429                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1430                            const lhs_vi = try isel.use(bin_op.lhs);
 1431                            const rhs_vi = try isel.use(bin_op.rhs);
 1432                            const lhs_mat = try lhs_vi.matReg(isel);
 1433                            const rhs_mat = try rhs_vi.matReg(isel);
 1434                            try isel.emit(.orr(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1435                            const skip_label = isel.instructions.items.len;
 1436                            try isel.emitPanic(.integer_overflow);
 1437                            try isel.emit(.@"b."(
 1438                                .invert(.ne),
 1439                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 1440                            ));
 1441                            try isel.emit(.ands(.wzr, lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1442                            try rhs_mat.finish(isel);
 1443                            try lhs_mat.finish(isel);
 1444                        },
 1445                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1446                    },
 1447                    .unsigned => switch (int_info.bits) {
 1448                        0 => unreachable,
 1449                        1 => {
 1450                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1451                            const lhs_vi = try isel.use(bin_op.lhs);
 1452                            const rhs_vi = try isel.use(bin_op.rhs);
 1453                            const lhs_mat = try lhs_vi.matReg(isel);
 1454                            const rhs_mat = try rhs_vi.matReg(isel);
 1455                            try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1456                            try rhs_mat.finish(isel);
 1457                            try lhs_mat.finish(isel);
 1458                        },
 1459                        2...16 => |bits| {
 1460                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1461                            const lhs_vi = try isel.use(bin_op.lhs);
 1462                            const rhs_vi = try isel.use(bin_op.rhs);
 1463                            const lhs_mat = try lhs_vi.matReg(isel);
 1464                            const rhs_mat = try rhs_vi.matReg(isel);
 1465                            const skip_label = isel.instructions.items.len;
 1466                            try isel.emitPanic(.integer_overflow);
 1467                            try isel.emit(.@"b."(
 1468                                .eq,
 1469                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 1470                            ));
 1471                            try isel.emit(.ands(.wzr, res_ra.w(), .{ .immediate = .{
 1472                                .N = .word,
 1473                                .immr = @intCast(32 - bits),
 1474                                .imms = @intCast(32 - bits - 1),
 1475                            } }));
 1476                            try isel.emit(.madd(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr));
 1477                            try rhs_mat.finish(isel);
 1478                            try lhs_mat.finish(isel);
 1479                        },
 1480                        17...32 => |bits| {
 1481                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1482                            const lhs_vi = try isel.use(bin_op.lhs);
 1483                            const rhs_vi = try isel.use(bin_op.rhs);
 1484                            const lhs_mat = try lhs_vi.matReg(isel);
 1485                            const rhs_mat = try rhs_vi.matReg(isel);
 1486                            const skip_label = isel.instructions.items.len;
 1487                            try isel.emitPanic(.integer_overflow);
 1488                            try isel.emit(.@"b."(
 1489                                .eq,
 1490                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 1491                            ));
 1492                            try isel.emit(.ands(.xzr, res_ra.x(), .{ .immediate = .{
 1493                                .N = .doubleword,
 1494                                .immr = @intCast(64 - bits),
 1495                                .imms = @intCast(64 - bits - 1),
 1496                            } }));
 1497                            try isel.emit(.umaddl(res_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr));
 1498                            try rhs_mat.finish(isel);
 1499                            try lhs_mat.finish(isel);
 1500                        },
 1501                        33...63 => |bits| {
 1502                            const lo64_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1503                            const lhs_vi = try isel.use(bin_op.lhs);
 1504                            const rhs_vi = try isel.use(bin_op.rhs);
 1505                            const lhs_mat = try lhs_vi.matReg(isel);
 1506                            const rhs_mat = try rhs_vi.matReg(isel);
 1507                            const hi64_ra = hi64_ra: {
 1508                                const lo64_lock = isel.tryLockReg(lo64_ra);
 1509                                defer lo64_lock.unlock(isel);
 1510                                break :hi64_ra try isel.allocIntReg();
 1511                            };
 1512                            defer isel.freeReg(hi64_ra);
 1513                            const skip_label = isel.instructions.items.len;
 1514                            try isel.emitPanic(.integer_overflow);
 1515                            try isel.emit(.cbz(
 1516                                hi64_ra.x(),
 1517                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 1518                            ));
 1519                            try isel.emit(.orr(hi64_ra.x(), hi64_ra.x(), .{ .shifted_register = .{
 1520                                .register = lo64_ra.x(),
 1521                                .shift = .{ .lsr = @intCast(bits) },
 1522                            } }));
 1523                            try isel.emit(.madd(lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
 1524                            try isel.emit(.umulh(hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
 1525                            try rhs_mat.finish(isel);
 1526                            try lhs_mat.finish(isel);
 1527                        },
 1528                        64 => {
 1529                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1530                            const lhs_vi = try isel.use(bin_op.lhs);
 1531                            const rhs_vi = try isel.use(bin_op.rhs);
 1532                            const lhs_mat = try lhs_vi.matReg(isel);
 1533                            const rhs_mat = try rhs_vi.matReg(isel);
 1534                            try isel.emit(.madd(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
 1535                            const hi64_ra = try isel.allocIntReg();
 1536                            defer isel.freeReg(hi64_ra);
 1537                            const skip_label = isel.instructions.items.len;
 1538                            try isel.emitPanic(.integer_overflow);
 1539                            try isel.emit(.cbz(
 1540                                hi64_ra.x(),
 1541                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 1542                            ));
 1543                            try isel.emit(.umulh(hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
 1544                            try rhs_mat.finish(isel);
 1545                            try lhs_mat.finish(isel);
 1546                        },
 1547                        65...128 => return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1548                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1549                    },
 1550                }
 1551            }
 1552            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1553        },
 1554        .mul_sat => |air_tag| {
 1555            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1556                defer res_vi.value.deref(isel);
 1557
 1558                const bin_op = air.data(air.inst_index).bin_op;
 1559                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1560                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 1561                const int_info = ty.intInfo(zcu);
 1562                switch (int_info.bits) {
 1563                    0 => unreachable,
 1564                    1 => {
 1565                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1566                        switch (int_info.signedness) {
 1567                            .signed => try isel.emit(.orr(res_ra.w(), .wzr, .{ .register = .wzr })),
 1568                            .unsigned => {
 1569                                const lhs_vi = try isel.use(bin_op.lhs);
 1570                                const rhs_vi = try isel.use(bin_op.rhs);
 1571                                const lhs_mat = try lhs_vi.matReg(isel);
 1572                                const rhs_mat = try rhs_vi.matReg(isel);
 1573                                try isel.emit(.@"and"(res_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1574                                try rhs_mat.finish(isel);
 1575                                try lhs_mat.finish(isel);
 1576                            },
 1577                        }
 1578                    },
 1579                    2...32 => |bits| {
 1580                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1581                        const saturated_ra = switch (int_info.signedness) {
 1582                            .signed => try isel.allocIntReg(),
 1583                            .unsigned => switch (bits) {
 1584                                else => unreachable,
 1585                                2...31 => try isel.allocIntReg(),
 1586                                32 => .zr,
 1587                            },
 1588                        };
 1589                        defer if (saturated_ra != .zr) isel.freeReg(saturated_ra);
 1590                        const unwrapped_ra = try isel.allocIntReg();
 1591                        defer isel.freeReg(unwrapped_ra);
 1592                        try isel.emit(switch (saturated_ra) {
 1593                            else => .csel(res_ra.w(), unwrapped_ra.w(), saturated_ra.w(), .eq),
 1594                            .zr => .csinv(res_ra.w(), unwrapped_ra.w(), saturated_ra.w(), .eq),
 1595                        });
 1596                        switch (bits) {
 1597                            else => unreachable,
 1598                            2...7, 9...15, 17...31 => switch (int_info.signedness) {
 1599                                .signed => {
 1600                                    const wrapped_ra = try isel.allocIntReg();
 1601                                    defer isel.freeReg(wrapped_ra);
 1602                                    switch (bits) {
 1603                                        else => unreachable,
 1604                                        1...7, 9...15 => {
 1605                                            try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .register = wrapped_ra.w() }));
 1606                                            try isel.emit(.sbfm(wrapped_ra.w(), unwrapped_ra.w(), .{
 1607                                                .N = .word,
 1608                                                .immr = 0,
 1609                                                .imms = @intCast(bits - 1),
 1610                                            }));
 1611                                        },
 1612                                        17...31 => {
 1613                                            try isel.emit(.subs(.xzr, unwrapped_ra.x(), .{ .register = wrapped_ra.x() }));
 1614                                            try isel.emit(.sbfm(wrapped_ra.x(), unwrapped_ra.x(), .{
 1615                                                .N = .doubleword,
 1616                                                .immr = 0,
 1617                                                .imms = @intCast(bits - 1),
 1618                                            }));
 1619                                        },
 1620                                    }
 1621                                },
 1622                                .unsigned => switch (bits) {
 1623                                    else => unreachable,
 1624                                    1...7, 9...15 => try isel.emit(.ands(.wzr, unwrapped_ra.w(), .{ .immediate = .{
 1625                                        .N = .word,
 1626                                        .immr = @intCast(32 - bits),
 1627                                        .imms = @intCast(32 - bits - 1),
 1628                                    } })),
 1629                                    17...31 => try isel.emit(.ands(.xzr, unwrapped_ra.x(), .{ .immediate = .{
 1630                                        .N = .doubleword,
 1631                                        .immr = @intCast(64 - bits),
 1632                                        .imms = @intCast(64 - bits - 1),
 1633                                    } })),
 1634                                },
 1635                            },
 1636                            8 => try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .extended_register = .{
 1637                                .register = unwrapped_ra.w(),
 1638                                .extend = switch (int_info.signedness) {
 1639                                    .signed => .{ .sxtb = 0 },
 1640                                    .unsigned => .{ .uxtb = 0 },
 1641                                },
 1642                            } })),
 1643                            16 => try isel.emit(.subs(.wzr, unwrapped_ra.w(), .{ .extended_register = .{
 1644                                .register = unwrapped_ra.w(),
 1645                                .extend = switch (int_info.signedness) {
 1646                                    .signed => .{ .sxth = 0 },
 1647                                    .unsigned => .{ .uxth = 0 },
 1648                                },
 1649                            } })),
 1650                            32 => try isel.emit(.subs(.xzr, unwrapped_ra.x(), .{ .extended_register = .{
 1651                                .register = unwrapped_ra.w(),
 1652                                .extend = switch (int_info.signedness) {
 1653                                    .signed => .{ .sxtw = 0 },
 1654                                    .unsigned => .{ .uxtw = 0 },
 1655                                },
 1656                            } })),
 1657                        }
 1658                        const lhs_vi = try isel.use(bin_op.lhs);
 1659                        const rhs_vi = try isel.use(bin_op.rhs);
 1660                        const lhs_mat = try lhs_vi.matReg(isel);
 1661                        const rhs_mat = try rhs_vi.matReg(isel);
 1662                        switch (int_info.signedness) {
 1663                            .signed => {
 1664                                try isel.emit(.eor(saturated_ra.w(), saturated_ra.w(), .{ .immediate = .{
 1665                                    .N = .word,
 1666                                    .immr = 0,
 1667                                    .imms = @intCast(bits - 1 - 1),
 1668                                } }));
 1669                                try isel.emit(.sbfm(saturated_ra.w(), saturated_ra.w(), .{
 1670                                    .N = .word,
 1671                                    .immr = @intCast(bits - 1),
 1672                                    .imms = @intCast(bits - 1 + 1 - 1),
 1673                                }));
 1674                                try isel.emit(.eor(saturated_ra.w(), lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 1675                            },
 1676                            .unsigned => switch (bits) {
 1677                                else => unreachable,
 1678                                2...31 => try isel.movImmediate(saturated_ra.w(), @as(u32, std.math.maxInt(u32)) >> @intCast(32 - bits)),
 1679                                32 => {},
 1680                            },
 1681                        }
 1682                        switch (bits) {
 1683                            else => unreachable,
 1684                            2...16 => try isel.emit(.madd(unwrapped_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), .wzr)),
 1685                            17...32 => switch (int_info.signedness) {
 1686                                .signed => try isel.emit(.smaddl(unwrapped_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr)),
 1687                                .unsigned => try isel.emit(.umaddl(unwrapped_ra.x(), lhs_mat.ra.w(), rhs_mat.ra.w(), .xzr)),
 1688                            },
 1689                        }
 1690                        try rhs_mat.finish(isel);
 1691                        try lhs_mat.finish(isel);
 1692                    },
 1693                    33...64 => |bits| {
 1694                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1695                        const saturated_ra = switch (int_info.signedness) {
 1696                            .signed => try isel.allocIntReg(),
 1697                            .unsigned => switch (bits) {
 1698                                else => unreachable,
 1699                                33...63 => try isel.allocIntReg(),
 1700                                64 => .zr,
 1701                            },
 1702                        };
 1703                        defer if (saturated_ra != .zr) isel.freeReg(saturated_ra);
 1704                        const unwrapped_lo64_ra = try isel.allocIntReg();
 1705                        defer isel.freeReg(unwrapped_lo64_ra);
 1706                        const unwrapped_hi64_ra = try isel.allocIntReg();
 1707                        defer isel.freeReg(unwrapped_hi64_ra);
 1708                        try isel.emit(switch (saturated_ra) {
 1709                            else => .csel(res_ra.x(), unwrapped_lo64_ra.x(), saturated_ra.x(), .eq),
 1710                            .zr => .csinv(res_ra.x(), unwrapped_lo64_ra.x(), saturated_ra.x(), .eq),
 1711                        });
 1712                        switch (int_info.signedness) {
 1713                            .signed => switch (bits) {
 1714                                else => unreachable,
 1715                                32...63 => {
 1716                                    const wrapped_lo64_ra = try isel.allocIntReg();
 1717                                    defer isel.freeReg(wrapped_lo64_ra);
 1718                                    try isel.emit(.ccmp(
 1719                                        unwrapped_lo64_ra.x(),
 1720                                        .{ .register = wrapped_lo64_ra.x() },
 1721                                        .{ .n = false, .z = false, .c = false, .v = false },
 1722                                        .eq,
 1723                                    ));
 1724                                    try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .shifted_register = .{
 1725                                        .register = unwrapped_lo64_ra.x(),
 1726                                        .shift = .{ .asr = 63 },
 1727                                    } }));
 1728                                    try isel.emit(.sbfm(wrapped_lo64_ra.x(), unwrapped_lo64_ra.x(), .{
 1729                                        .N = .doubleword,
 1730                                        .immr = 0,
 1731                                        .imms = @intCast(bits - 1),
 1732                                    }));
 1733                                },
 1734                                64 => try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .shifted_register = .{
 1735                                    .register = unwrapped_lo64_ra.x(),
 1736                                    .shift = .{ .asr = @intCast(bits - 1) },
 1737                                } })),
 1738                            },
 1739                            .unsigned => switch (bits) {
 1740                                else => unreachable,
 1741                                32...63 => {
 1742                                    const overflow_ra = try isel.allocIntReg();
 1743                                    defer isel.freeReg(overflow_ra);
 1744                                    try isel.emit(.subs(.xzr, overflow_ra.x(), .{ .immediate = 0 }));
 1745                                    try isel.emit(.orr(overflow_ra.x(), unwrapped_hi64_ra.x(), .{ .shifted_register = .{
 1746                                        .register = unwrapped_lo64_ra.x(),
 1747                                        .shift = .{ .lsr = @intCast(bits) },
 1748                                    } }));
 1749                                },
 1750                                64 => try isel.emit(.subs(.xzr, unwrapped_hi64_ra.x(), .{ .immediate = 0 })),
 1751                            },
 1752                        }
 1753                        const lhs_vi = try isel.use(bin_op.lhs);
 1754                        const rhs_vi = try isel.use(bin_op.rhs);
 1755                        const lhs_mat = try lhs_vi.matReg(isel);
 1756                        const rhs_mat = try rhs_vi.matReg(isel);
 1757                        switch (int_info.signedness) {
 1758                            .signed => {
 1759                                try isel.emit(.eor(saturated_ra.x(), saturated_ra.x(), .{ .immediate = .{
 1760                                    .N = .doubleword,
 1761                                    .immr = 0,
 1762                                    .imms = @intCast(bits - 1 - 1),
 1763                                } }));
 1764                                try isel.emit(.sbfm(saturated_ra.x(), saturated_ra.x(), .{
 1765                                    .N = .doubleword,
 1766                                    .immr = @intCast(bits - 1),
 1767                                    .imms = @intCast(bits - 1 + 1 - 1),
 1768                                }));
 1769                                try isel.emit(.eor(saturated_ra.x(), lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
 1770                                try isel.emit(.madd(unwrapped_lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
 1771                                try isel.emit(.smulh(unwrapped_hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
 1772                            },
 1773                            .unsigned => {
 1774                                switch (bits) {
 1775                                    else => unreachable,
 1776                                    32...63 => try isel.movImmediate(saturated_ra.x(), @as(u64, std.math.maxInt(u64)) >> @intCast(64 - bits)),
 1777                                    64 => {},
 1778                                }
 1779                                try isel.emit(.madd(unwrapped_lo64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), .xzr));
 1780                                try isel.emit(.umulh(unwrapped_hi64_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()));
 1781                            },
 1782                        }
 1783                        try rhs_mat.finish(isel);
 1784                        try lhs_mat.finish(isel);
 1785                    },
 1786                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 1787                }
 1788            }
 1789            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1790        },
 1791        .div_float, .div_float_optimized => {
 1792            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1793                defer res_vi.value.deref(isel);
 1794
 1795                const bin_op = air.data(air.inst_index).bin_op;
 1796                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1797                switch (ty.floatBits(isel.target)) {
 1798                    else => unreachable,
 1799                    16, 32, 64 => |bits| {
 1800                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1801                        const need_fcvt = switch (bits) {
 1802                            else => unreachable,
 1803                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 1804                            32, 64 => false,
 1805                        };
 1806                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 1807                        const lhs_vi = try isel.use(bin_op.lhs);
 1808                        const rhs_vi = try isel.use(bin_op.rhs);
 1809                        const lhs_mat = try lhs_vi.matReg(isel);
 1810                        const rhs_mat = try rhs_vi.matReg(isel);
 1811                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 1812                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 1813                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 1814                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 1815                        try isel.emit(bits: switch (bits) {
 1816                            else => unreachable,
 1817                            16 => if (need_fcvt)
 1818                                continue :bits 32
 1819                            else
 1820                                .fdiv(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 1821                            32 => .fdiv(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 1822                            64 => .fdiv(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 1823                        });
 1824                        if (need_fcvt) {
 1825                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 1826                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 1827                        }
 1828                        try rhs_mat.finish(isel);
 1829                        try lhs_mat.finish(isel);
 1830                    },
 1831                    80, 128 => |bits| {
 1832                        try call.prepareReturn(isel);
 1833                        switch (bits) {
 1834                            else => unreachable,
 1835                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 1836                            80 => {
 1837                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 1838                                const res_hi16_vi = try res_hi16_it.only(isel);
 1839                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 1840                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 1841                                const res_lo64_vi = try res_lo64_it.only(isel);
 1842                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 1843                            },
 1844                        }
 1845                        try call.finishReturn(isel);
 1846
 1847                        try call.prepareCallee(isel);
 1848                        try isel.global_relocs.append(gpa, .{
 1849                            .name = switch (bits) {
 1850                                else => unreachable,
 1851                                16 => "__divhf3",
 1852                                32 => "__divsf3",
 1853                                64 => "__divdf3",
 1854                                80 => "__divxf3",
 1855                                128 => "__divtf3",
 1856                            },
 1857                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 1858                        });
 1859                        try isel.emit(.bl(0));
 1860                        try call.finishCallee(isel);
 1861
 1862                        try call.prepareParams(isel);
 1863                        const lhs_vi = try isel.use(bin_op.lhs);
 1864                        const rhs_vi = try isel.use(bin_op.rhs);
 1865                        switch (bits) {
 1866                            else => unreachable,
 1867                            16, 32, 64, 128 => {
 1868                                try call.paramLiveOut(isel, rhs_vi, .v1);
 1869                                try call.paramLiveOut(isel, lhs_vi, .v0);
 1870                            },
 1871                            80 => {
 1872                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 1873                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 1874                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 1875                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 1876                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 1877                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 1878                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 1879                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 1880                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 1881                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 1882                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 1883                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 1884                            },
 1885                        }
 1886                        try call.finishParams(isel);
 1887                    },
 1888                }
 1889            }
 1890            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 1891        },
 1892        .div_trunc, .div_trunc_optimized, .div_floor, .div_floor_optimized, .div_exact, .div_exact_optimized => |air_tag| {
 1893            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 1894                defer res_vi.value.deref(isel);
 1895
 1896                const bin_op = air.data(air.inst_index).bin_op;
 1897                const ty = isel.air.typeOf(bin_op.lhs, ip);
 1898                if (!ty.isRuntimeFloat()) {
 1899                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 1900                    const int_info = ty.intInfo(zcu);
 1901                    switch (int_info.bits) {
 1902                        0 => unreachable,
 1903                        1...64 => |bits| {
 1904                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 1905                            const lhs_vi = try isel.use(bin_op.lhs);
 1906                            const rhs_vi = try isel.use(bin_op.rhs);
 1907                            const lhs_mat = try lhs_vi.matReg(isel);
 1908                            const rhs_mat = try rhs_vi.matReg(isel);
 1909                            const div_ra = div_ra: switch (air_tag) {
 1910                                else => unreachable,
 1911                                .div_trunc, .div_exact => res_ra,
 1912                                .div_floor => switch (int_info.signedness) {
 1913                                    .signed => {
 1914                                        const div_ra = try isel.allocIntReg();
 1915                                        errdefer isel.freeReg(div_ra);
 1916                                        const rem_ra = try isel.allocIntReg();
 1917                                        defer isel.freeReg(rem_ra);
 1918                                        switch (bits) {
 1919                                            else => unreachable,
 1920                                            1...32 => {
 1921                                                try isel.emit(.csel(res_ra.w(), div_ra.w(), rem_ra.w(), .pl));
 1922                                                try isel.emit(.sub(rem_ra.w(), div_ra.w(), .{ .immediate = 1 }));
 1923                                                try isel.emit(.ccmp(
 1924                                                    rem_ra.w(),
 1925                                                    .{ .immediate = 0 },
 1926                                                    .{ .n = false, .z = false, .c = false, .v = false },
 1927                                                    .ne,
 1928                                                ));
 1929                                                try isel.emit(.eor(rem_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
 1930                                                try isel.emit(.subs(.wzr, rem_ra.w(), .{ .immediate = 0 }));
 1931                                                try isel.emit(.msub(rem_ra.w(), div_ra.w(), rhs_mat.ra.w(), lhs_mat.ra.w()));
 1932                                            },
 1933                                            33...64 => {
 1934                                                try isel.emit(.csel(res_ra.x(), div_ra.x(), rem_ra.x(), .pl));
 1935                                                try isel.emit(.sub(rem_ra.x(), div_ra.x(), .{ .immediate = 1 }));
 1936                                                try isel.emit(.ccmp(
 1937                                                    rem_ra.x(),
 1938                                                    .{ .immediate = 0 },
 1939                                                    .{ .n = false, .z = false, .c = false, .v = false },
 1940                                                    .ne,
 1941                                                ));
 1942                                                try isel.emit(.eor(rem_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
 1943                                                try isel.emit(.subs(.xzr, rem_ra.x(), .{ .immediate = 0 }));
 1944                                                try isel.emit(.msub(rem_ra.x(), div_ra.x(), rhs_mat.ra.x(), lhs_mat.ra.x()));
 1945                                            },
 1946                                        }
 1947                                        break :div_ra div_ra;
 1948                                    },
 1949                                    .unsigned => res_ra,
 1950                                },
 1951                            };
 1952                            defer if (div_ra != res_ra) isel.freeReg(div_ra);
 1953                            try isel.emit(switch (bits) {
 1954                                else => unreachable,
 1955                                1...32 => switch (int_info.signedness) {
 1956                                    .signed => .sdiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 1957                                    .unsigned => .udiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 1958                                },
 1959                                33...64 => switch (int_info.signedness) {
 1960                                    .signed => .sdiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 1961                                    .unsigned => .udiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 1962                                },
 1963                            });
 1964                            try rhs_mat.finish(isel);
 1965                            try lhs_mat.finish(isel);
 1966                        },
 1967                        65...128 => {
 1968                            switch (air_tag) {
 1969                                else => unreachable,
 1970                                .div_trunc, .div_exact => {},
 1971                                .div_floor => switch (int_info.signedness) {
 1972                                    .signed => return isel.fail("unimplemented {t}", .{air_tag}),
 1973                                    .unsigned => {},
 1974                                },
 1975                            }
 1976
 1977                            try call.prepareReturn(isel);
 1978                            var res_hi64_it = res_vi.value.field(ty, 8, 8);
 1979                            const res_hi64_vi = try res_hi64_it.only(isel);
 1980                            try call.returnLiveIn(isel, res_hi64_vi.?, .r1);
 1981                            var res_lo64_it = res_vi.value.field(ty, 0, 8);
 1982                            const res_lo64_vi = try res_lo64_it.only(isel);
 1983                            try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 1984                            try call.finishReturn(isel);
 1985
 1986                            try call.prepareCallee(isel);
 1987                            try isel.global_relocs.append(gpa, .{
 1988                                .name = switch (int_info.signedness) {
 1989                                    .signed => "__divti3",
 1990                                    .unsigned => "__udivti3",
 1991                                },
 1992                                .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 1993                            });
 1994                            try isel.emit(.bl(0));
 1995                            try call.finishCallee(isel);
 1996
 1997                            try call.prepareParams(isel);
 1998                            const lhs_vi = try isel.use(bin_op.lhs);
 1999                            const rhs_vi = try isel.use(bin_op.rhs);
 2000                            var rhs_hi64_it = rhs_vi.field(ty, 8, 8);
 2001                            const rhs_hi64_vi = try rhs_hi64_it.only(isel);
 2002                            try call.paramLiveOut(isel, rhs_hi64_vi.?, .r3);
 2003                            var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 2004                            const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 2005                            try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 2006                            var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
 2007                            const lhs_hi64_vi = try lhs_hi64_it.only(isel);
 2008                            try call.paramLiveOut(isel, lhs_hi64_vi.?, .r1);
 2009                            var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 2010                            const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 2011                            try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 2012                            try call.finishParams(isel);
 2013                        },
 2014                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 2015                    }
 2016                } else switch (ty.floatBits(isel.target)) {
 2017                    else => unreachable,
 2018                    16, 32, 64 => |bits| {
 2019                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2020                        const need_fcvt = switch (bits) {
 2021                            else => unreachable,
 2022                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 2023                            32, 64 => false,
 2024                        };
 2025                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 2026                        const lhs_vi = try isel.use(bin_op.lhs);
 2027                        const rhs_vi = try isel.use(bin_op.rhs);
 2028                        const lhs_mat = try lhs_vi.matReg(isel);
 2029                        const rhs_mat = try rhs_vi.matReg(isel);
 2030                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 2031                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 2032                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 2033                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 2034                        bits: switch (bits) {
 2035                            else => unreachable,
 2036                            16 => if (need_fcvt) continue :bits 32 else {
 2037                                switch (air_tag) {
 2038                                    else => unreachable,
 2039                                    .div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.h(), res_ra.h())),
 2040                                    .div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.h(), res_ra.h())),
 2041                                    .div_exact, .div_exact_optimized => {},
 2042                                }
 2043                                try isel.emit(.fdiv(res_ra.h(), lhs_ra.h(), rhs_ra.h()));
 2044                            },
 2045                            32 => {
 2046                                switch (air_tag) {
 2047                                    else => unreachable,
 2048                                    .div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.s(), res_ra.s())),
 2049                                    .div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.s(), res_ra.s())),
 2050                                    .div_exact, .div_exact_optimized => {},
 2051                                }
 2052                                try isel.emit(.fdiv(res_ra.s(), lhs_ra.s(), rhs_ra.s()));
 2053                            },
 2054                            64 => {
 2055                                switch (air_tag) {
 2056                                    else => unreachable,
 2057                                    .div_trunc, .div_trunc_optimized => try isel.emit(.frintz(res_ra.d(), res_ra.d())),
 2058                                    .div_floor, .div_floor_optimized => try isel.emit(.frintm(res_ra.d(), res_ra.d())),
 2059                                    .div_exact, .div_exact_optimized => {},
 2060                                }
 2061                                try isel.emit(.fdiv(res_ra.d(), lhs_ra.d(), rhs_ra.d()));
 2062                            },
 2063                        }
 2064                        if (need_fcvt) {
 2065                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 2066                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 2067                        }
 2068                        try rhs_mat.finish(isel);
 2069                        try lhs_mat.finish(isel);
 2070                    },
 2071                    80, 128 => |bits| {
 2072                        try call.prepareReturn(isel);
 2073                        switch (bits) {
 2074                            else => unreachable,
 2075                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 2076                            80 => {
 2077                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 2078                                const res_hi16_vi = try res_hi16_it.only(isel);
 2079                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 2080                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 2081                                const res_lo64_vi = try res_lo64_it.only(isel);
 2082                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 2083                            },
 2084                        }
 2085                        try call.finishReturn(isel);
 2086
 2087                        try call.prepareCallee(isel);
 2088                        switch (air_tag) {
 2089                            else => unreachable,
 2090                            .div_trunc, .div_trunc_optimized => {
 2091                                try isel.global_relocs.append(gpa, .{
 2092                                    .name = switch (bits) {
 2093                                        else => unreachable,
 2094                                        16 => "__trunch",
 2095                                        32 => "truncf",
 2096                                        64 => "trunc",
 2097                                        80 => "__truncx",
 2098                                        128 => "truncq",
 2099                                    },
 2100                                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2101                                });
 2102                                try isel.emit(.bl(0));
 2103                            },
 2104                            .div_floor, .div_floor_optimized => {
 2105                                try isel.global_relocs.append(gpa, .{
 2106                                    .name = switch (bits) {
 2107                                        else => unreachable,
 2108                                        16 => "__floorh",
 2109                                        32 => "floorf",
 2110                                        64 => "floor",
 2111                                        80 => "__floorx",
 2112                                        128 => "floorq",
 2113                                    },
 2114                                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2115                                });
 2116                                try isel.emit(.bl(0));
 2117                            },
 2118                            .div_exact, .div_exact_optimized => {},
 2119                        }
 2120                        try isel.global_relocs.append(gpa, .{
 2121                            .name = switch (bits) {
 2122                                else => unreachable,
 2123                                16 => "__divhf3",
 2124                                32 => "__divsf3",
 2125                                64 => "__divdf3",
 2126                                80 => "__divxf3",
 2127                                128 => "__divtf3",
 2128                            },
 2129                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2130                        });
 2131                        try isel.emit(.bl(0));
 2132                        try call.finishCallee(isel);
 2133
 2134                        try call.prepareParams(isel);
 2135                        const lhs_vi = try isel.use(bin_op.lhs);
 2136                        const rhs_vi = try isel.use(bin_op.rhs);
 2137                        switch (bits) {
 2138                            else => unreachable,
 2139                            16, 32, 64, 128 => {
 2140                                try call.paramLiveOut(isel, rhs_vi, .v1);
 2141                                try call.paramLiveOut(isel, lhs_vi, .v0);
 2142                            },
 2143                            80 => {
 2144                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 2145                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 2146                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 2147                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 2148                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 2149                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 2150                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 2151                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 2152                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 2153                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 2154                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 2155                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 2156                            },
 2157                        }
 2158                        try call.finishParams(isel);
 2159                    },
 2160                }
 2161            }
 2162            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2163        },
 2164        .rem, .rem_optimized, .mod, .mod_optimized => |air_tag| {
 2165            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 2166                defer res_vi.value.deref(isel);
 2167
 2168                const bin_op = air.data(air.inst_index).bin_op;
 2169                const ty = isel.air.typeOf(bin_op.lhs, ip);
 2170                if (!ty.isRuntimeFloat()) {
 2171                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2172                    const int_info = ty.intInfo(zcu);
 2173                    if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2174
 2175                    const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2176                    const lhs_vi = try isel.use(bin_op.lhs);
 2177                    const rhs_vi = try isel.use(bin_op.rhs);
 2178                    const lhs_mat = try lhs_vi.matReg(isel);
 2179                    const rhs_mat = try rhs_vi.matReg(isel);
 2180                    const div_ra = try isel.allocIntReg();
 2181                    defer isel.freeReg(div_ra);
 2182                    const rem_ra = rem_ra: switch (air_tag) {
 2183                        else => unreachable,
 2184                        .rem => res_ra,
 2185                        .mod => switch (int_info.signedness) {
 2186                            .signed => {
 2187                                const rem_ra = try isel.allocIntReg();
 2188                                errdefer isel.freeReg(rem_ra);
 2189                                switch (int_info.bits) {
 2190                                    else => unreachable,
 2191                                    1...32 => {
 2192                                        try isel.emit(.csel(res_ra.w(), rem_ra.w(), div_ra.w(), .pl));
 2193                                        try isel.emit(.add(div_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
 2194                                        try isel.emit(.ccmp(
 2195                                            div_ra.w(),
 2196                                            .{ .immediate = 0 },
 2197                                            .{ .n = false, .z = false, .c = false, .v = false },
 2198                                            .ne,
 2199                                        ));
 2200                                        try isel.emit(.eor(div_ra.w(), rem_ra.w(), .{ .register = rhs_mat.ra.w() }));
 2201                                        try isel.emit(.subs(.wzr, rem_ra.w(), .{ .immediate = 0 }));
 2202                                    },
 2203                                    33...64 => {
 2204                                        try isel.emit(.csel(res_ra.x(), rem_ra.x(), div_ra.x(), .pl));
 2205                                        try isel.emit(.add(div_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
 2206                                        try isel.emit(.ccmp(
 2207                                            div_ra.x(),
 2208                                            .{ .immediate = 0 },
 2209                                            .{ .n = false, .z = false, .c = false, .v = false },
 2210                                            .ne,
 2211                                        ));
 2212                                        try isel.emit(.eor(div_ra.x(), rem_ra.x(), .{ .register = rhs_mat.ra.x() }));
 2213                                        try isel.emit(.subs(.xzr, rem_ra.x(), .{ .immediate = 0 }));
 2214                                    },
 2215                                }
 2216                                break :rem_ra rem_ra;
 2217                            },
 2218                            .unsigned => res_ra,
 2219                        },
 2220                    };
 2221                    defer if (rem_ra != res_ra) isel.freeReg(rem_ra);
 2222                    switch (int_info.bits) {
 2223                        else => unreachable,
 2224                        1...32 => {
 2225                            try isel.emit(.msub(rem_ra.w(), div_ra.w(), rhs_mat.ra.w(), lhs_mat.ra.w()));
 2226                            try isel.emit(switch (int_info.signedness) {
 2227                                .signed => .sdiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 2228                                .unsigned => .udiv(div_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 2229                            });
 2230                        },
 2231                        33...64 => {
 2232                            try isel.emit(.msub(rem_ra.x(), div_ra.x(), rhs_mat.ra.x(), lhs_mat.ra.x()));
 2233                            try isel.emit(switch (int_info.signedness) {
 2234                                .signed => .sdiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 2235                                .unsigned => .udiv(div_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 2236                            });
 2237                        },
 2238                    }
 2239                    try rhs_mat.finish(isel);
 2240                    try lhs_mat.finish(isel);
 2241                } else {
 2242                    const bits = ty.floatBits(isel.target);
 2243                    switch (air_tag) {
 2244                        else => unreachable,
 2245                        .rem, .rem_optimized => {
 2246                            if (!res_vi.value.isUsed(isel)) break :unused;
 2247                            try call.prepareReturn(isel);
 2248                            switch (bits) {
 2249                                else => unreachable,
 2250                                16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 2251                                80 => {
 2252                                    var res_hi16_it = res_vi.value.field(ty, 8, 8);
 2253                                    const res_hi16_vi = try res_hi16_it.only(isel);
 2254                                    try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 2255                                    var res_lo64_it = res_vi.value.field(ty, 0, 8);
 2256                                    const res_lo64_vi = try res_lo64_it.only(isel);
 2257                                    try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 2258                                },
 2259                            }
 2260                            try call.finishReturn(isel);
 2261                        },
 2262                        .mod, .mod_optimized => switch (bits) {
 2263                            else => unreachable,
 2264                            16, 32, 64 => {
 2265                                const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2266                                try call.prepareReturn(isel);
 2267                                const rem_ra: Register.Alias = .v0;
 2268                                const temp1_ra: Register.Alias = .v1;
 2269                                const temp2_ra: Register.Alias = switch (res_ra) {
 2270                                    rem_ra, temp1_ra => .v2,
 2271                                    else => res_ra,
 2272                                };
 2273                                const need_fcvt = switch (bits) {
 2274                                    else => unreachable,
 2275                                    16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 2276                                    32, 64 => false,
 2277                                };
 2278                                if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 2279                                try isel.emit(switch (res_ra) {
 2280                                    rem_ra => .bif(res_ra.@"8b"(), temp2_ra.@"8b"(), temp1_ra.@"8b"()),
 2281                                    temp1_ra => .bsl(res_ra.@"8b"(), rem_ra.@"8b"(), temp2_ra.@"8b"()),
 2282                                    else => .bit(res_ra.@"8b"(), rem_ra.@"8b"(), temp1_ra.@"8b"()),
 2283                                });
 2284                                const rhs_vi = try isel.use(bin_op.rhs);
 2285                                const rhs_mat = try rhs_vi.matReg(isel);
 2286                                try isel.emit(bits: switch (bits) {
 2287                                    else => unreachable,
 2288                                    16 => if (need_fcvt)
 2289                                        continue :bits 32
 2290                                    else
 2291                                        .fadd(temp2_ra.h(), rem_ra.h(), rhs_mat.ra.h()),
 2292                                    32 => .fadd(temp2_ra.s(), rem_ra.s(), rhs_mat.ra.s()),
 2293                                    64 => .fadd(temp2_ra.d(), rem_ra.d(), rhs_mat.ra.d()),
 2294                                });
 2295                                if (need_fcvt) {
 2296                                    try isel.emit(.fcvt(rhs_mat.ra.s(), rhs_mat.ra.h()));
 2297                                    try isel.emit(.fcvt(rem_ra.s(), rem_ra.h()));
 2298                                }
 2299                                try isel.emit(.orr(temp1_ra.@"8b"(), temp1_ra.@"8b"(), .{
 2300                                    .register = temp2_ra.@"8b"(),
 2301                                }));
 2302                                try isel.emit(switch (bits) {
 2303                                    else => unreachable,
 2304                                    16 => .cmge(temp1_ra.@"4h"(), temp1_ra.@"4h"(), .zero),
 2305                                    32 => .cmge(temp1_ra.@"2s"(), temp1_ra.@"2s"(), .zero),
 2306                                    64 => .cmge(temp1_ra.d(), temp1_ra.d(), .zero),
 2307                                });
 2308                                try isel.emit(switch (bits) {
 2309                                    else => unreachable,
 2310                                    16 => .fcmeq(temp2_ra.h(), rem_ra.h(), .zero),
 2311                                    32 => .fcmeq(temp2_ra.s(), rem_ra.s(), .zero),
 2312                                    64 => .fcmeq(temp2_ra.d(), rem_ra.d(), .zero),
 2313                                });
 2314                                try isel.emit(.eor(temp1_ra.@"8b"(), rem_ra.@"8b"(), .{
 2315                                    .register = rhs_mat.ra.@"8b"(),
 2316                                }));
 2317                                try rhs_mat.finish(isel);
 2318                                try call.finishReturn(isel);
 2319                            },
 2320                            80, 128 => {
 2321                                if (!res_vi.value.isUsed(isel)) break :unused;
 2322                                try call.prepareReturn(isel);
 2323                                switch (bits) {
 2324                                    else => unreachable,
 2325                                    16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 2326                                    80 => {
 2327                                        var res_hi16_it = res_vi.value.field(ty, 8, 8);
 2328                                        const res_hi16_vi = try res_hi16_it.only(isel);
 2329                                        try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 2330                                        var res_lo64_it = res_vi.value.field(ty, 0, 8);
 2331                                        const res_lo64_vi = try res_lo64_it.only(isel);
 2332                                        try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 2333                                    },
 2334                                }
 2335                                const skip_label = isel.instructions.items.len;
 2336                                try isel.global_relocs.append(gpa, .{
 2337                                    .name = switch (bits) {
 2338                                        else => unreachable,
 2339                                        16 => "__addhf3",
 2340                                        32 => "__addsf3",
 2341                                        64 => "__adddf3",
 2342                                        80 => "__addxf3",
 2343                                        128 => "__addtf3",
 2344                                    },
 2345                                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2346                                });
 2347                                try isel.emit(.bl(0));
 2348                                const rhs_vi = try isel.use(bin_op.rhs);
 2349                                switch (bits) {
 2350                                    else => unreachable,
 2351                                    80 => {
 2352                                        const lhs_lo64_ra: Register.Alias = .r0;
 2353                                        const lhs_hi16_ra: Register.Alias = .r1;
 2354                                        const rhs_lo64_ra: Register.Alias = .r2;
 2355                                        const rhs_hi16_ra: Register.Alias = .r3;
 2356                                        const temp_ra: Register.Alias = .r4;
 2357                                        var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 2358                                        const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 2359                                        try call.paramLiveOut(isel, rhs_hi16_vi.?, rhs_hi16_ra);
 2360                                        var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 2361                                        const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 2362                                        try call.paramLiveOut(isel, rhs_lo64_vi.?, rhs_lo64_ra);
 2363                                        try isel.emit(.cbz(
 2364                                            temp_ra.x(),
 2365                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 2366                                        ));
 2367                                        try isel.emit(.orr(temp_ra.x(), lhs_lo64_ra.x(), .{ .shifted_register = .{
 2368                                            .register = lhs_hi16_ra.x(),
 2369                                            .shift = .{ .lsl = 64 - 15 },
 2370                                        } }));
 2371                                        try isel.emit(.tbz(
 2372                                            temp_ra.w(),
 2373                                            15,
 2374                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 2375                                        ));
 2376                                        try isel.emit(.eor(temp_ra.w(), lhs_hi16_ra.w(), .{
 2377                                            .register = rhs_hi16_ra.w(),
 2378                                        }));
 2379                                    },
 2380                                    128 => {
 2381                                        const lhs_ra: Register.Alias = .v0;
 2382                                        const rhs_ra: Register.Alias = .v1;
 2383                                        const temp1_ra: Register.Alias = .r0;
 2384                                        const temp2_ra: Register.Alias = .r1;
 2385                                        try call.paramLiveOut(isel, rhs_vi, rhs_ra);
 2386                                        try isel.emit(.@"b."(
 2387                                            .pl,
 2388                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 2389                                        ));
 2390                                        try isel.emit(.cbz(
 2391                                            temp1_ra.x(),
 2392                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 2393                                        ));
 2394                                        try isel.emit(.orr(temp1_ra.x(), temp1_ra.x(), .{ .shifted_register = .{
 2395                                            .register = temp2_ra.x(),
 2396                                            .shift = .{ .lsl = 1 },
 2397                                        } }));
 2398                                        try isel.emit(.fmov(temp1_ra.x(), .{
 2399                                            .register = rhs_ra.d(),
 2400                                        }));
 2401                                        try isel.emit(.tbz(
 2402                                            temp1_ra.x(),
 2403                                            63,
 2404                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 2405                                        ));
 2406                                        try isel.emit(.eor(temp1_ra.x(), temp1_ra.x(), .{
 2407                                            .register = temp2_ra.x(),
 2408                                        }));
 2409                                        try isel.emit(.fmov(temp2_ra.x(), .{
 2410                                            .register = rhs_ra.@"d[]"(1),
 2411                                        }));
 2412                                        try isel.emit(.fmov(temp1_ra.x(), .{
 2413                                            .register = lhs_ra.@"d[]"(1),
 2414                                        }));
 2415                                    },
 2416                                }
 2417                                try call.finishReturn(isel);
 2418                            },
 2419                        },
 2420                    }
 2421
 2422                    try call.prepareCallee(isel);
 2423                    try isel.global_relocs.append(gpa, .{
 2424                        .name = switch (bits) {
 2425                            else => unreachable,
 2426                            16 => "__fmodh",
 2427                            32 => "fmodf",
 2428                            64 => "fmod",
 2429                            80 => "__fmodx",
 2430                            128 => "fmodq",
 2431                        },
 2432                        .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2433                    });
 2434                    try isel.emit(.bl(0));
 2435                    try call.finishCallee(isel);
 2436
 2437                    try call.prepareParams(isel);
 2438                    const lhs_vi = try isel.use(bin_op.lhs);
 2439                    const rhs_vi = try isel.use(bin_op.rhs);
 2440                    switch (bits) {
 2441                        else => unreachable,
 2442                        16, 32, 64, 128 => {
 2443                            try call.paramLiveOut(isel, rhs_vi, .v1);
 2444                            try call.paramLiveOut(isel, lhs_vi, .v0);
 2445                        },
 2446                        80 => {
 2447                            var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 2448                            const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 2449                            try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 2450                            var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 2451                            const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 2452                            try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 2453                            var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 2454                            const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 2455                            try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 2456                            var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 2457                            const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 2458                            try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 2459                        },
 2460                    }
 2461                    try call.finishParams(isel);
 2462                }
 2463            }
 2464            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2465        },
 2466        .ptr_add, .ptr_sub => |air_tag| {
 2467            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 2468                defer res_vi.value.deref(isel);
 2469                const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2470
 2471                const ty_pl = air.data(air.inst_index).ty_pl;
 2472                const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
 2473                const elem_size = ty_pl.ty.toType().elemType2(zcu).abiSize(zcu);
 2474
 2475                const base_vi = try isel.use(bin_op.lhs);
 2476                var base_part_it = base_vi.field(ty_pl.ty.toType(), 0, 8);
 2477                const base_part_vi = try base_part_it.only(isel);
 2478                const base_part_mat = try base_part_vi.?.matReg(isel);
 2479                const index_vi = try isel.use(bin_op.rhs);
 2480                try isel.elemPtr(res_ra, base_part_mat.ra, switch (air_tag) {
 2481                    else => unreachable,
 2482                    .ptr_add => .add,
 2483                    .ptr_sub => .sub,
 2484                }, elem_size, index_vi);
 2485                try base_part_mat.finish(isel);
 2486            }
 2487            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2488        },
 2489        .max, .min => |air_tag| {
 2490            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 2491                defer res_vi.value.deref(isel);
 2492
 2493                const bin_op = air.data(air.inst_index).bin_op;
 2494                const ty = isel.air.typeOf(bin_op.lhs, ip);
 2495                if (!ty.isRuntimeFloat()) {
 2496                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2497                    const int_info = ty.intInfo(zcu);
 2498                    if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2499
 2500                    const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2501                    const lhs_vi = try isel.use(bin_op.lhs);
 2502                    const rhs_vi = try isel.use(bin_op.rhs);
 2503                    const lhs_mat = try lhs_vi.matReg(isel);
 2504                    const rhs_mat = try rhs_vi.matReg(isel);
 2505                    const cond: codegen.aarch64.encoding.ConditionCode = switch (air_tag) {
 2506                        else => unreachable,
 2507                        .max => switch (int_info.signedness) {
 2508                            .signed => .ge,
 2509                            .unsigned => .hs,
 2510                        },
 2511                        .min => switch (int_info.signedness) {
 2512                            .signed => .lt,
 2513                            .unsigned => .lo,
 2514                        },
 2515                    };
 2516                    switch (int_info.bits) {
 2517                        else => unreachable,
 2518                        1...32 => {
 2519                            try isel.emit(.csel(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w(), cond));
 2520                            try isel.emit(.subs(.wzr, lhs_mat.ra.w(), .{ .register = rhs_mat.ra.w() }));
 2521                        },
 2522                        33...64 => {
 2523                            try isel.emit(.csel(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x(), cond));
 2524                            try isel.emit(.subs(.xzr, lhs_mat.ra.x(), .{ .register = rhs_mat.ra.x() }));
 2525                        },
 2526                    }
 2527                    try rhs_mat.finish(isel);
 2528                    try lhs_mat.finish(isel);
 2529                } else switch (ty.floatBits(isel.target)) {
 2530                    else => unreachable,
 2531                    16, 32, 64 => |bits| {
 2532                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2533                        const need_fcvt = switch (bits) {
 2534                            else => unreachable,
 2535                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 2536                            32, 64 => false,
 2537                        };
 2538                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 2539                        const lhs_vi = try isel.use(bin_op.lhs);
 2540                        const rhs_vi = try isel.use(bin_op.rhs);
 2541                        const lhs_mat = try lhs_vi.matReg(isel);
 2542                        const rhs_mat = try rhs_vi.matReg(isel);
 2543                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 2544                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 2545                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 2546                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 2547                        try isel.emit(bits: switch (bits) {
 2548                            else => unreachable,
 2549                            16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
 2550                                else => unreachable,
 2551                                .max => .fmaxnm(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 2552                                .min => .fminnm(res_ra.h(), lhs_ra.h(), rhs_ra.h()),
 2553                            },
 2554                            32 => switch (air_tag) {
 2555                                else => unreachable,
 2556                                .max => .fmaxnm(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 2557                                .min => .fminnm(res_ra.s(), lhs_ra.s(), rhs_ra.s()),
 2558                            },
 2559                            64 => switch (air_tag) {
 2560                                else => unreachable,
 2561                                .max => .fmaxnm(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 2562                                .min => .fminnm(res_ra.d(), lhs_ra.d(), rhs_ra.d()),
 2563                            },
 2564                        });
 2565                        if (need_fcvt) {
 2566                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 2567                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 2568                        }
 2569                        try rhs_mat.finish(isel);
 2570                        try lhs_mat.finish(isel);
 2571                    },
 2572                    80, 128 => |bits| {
 2573                        try call.prepareReturn(isel);
 2574                        switch (bits) {
 2575                            else => unreachable,
 2576                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 2577                            80 => {
 2578                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 2579                                const res_hi16_vi = try res_hi16_it.only(isel);
 2580                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 2581                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 2582                                const res_lo64_vi = try res_lo64_it.only(isel);
 2583                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 2584                            },
 2585                        }
 2586                        try call.finishReturn(isel);
 2587
 2588                        try call.prepareCallee(isel);
 2589                        try isel.global_relocs.append(gpa, .{
 2590                            .name = switch (air_tag) {
 2591                                else => unreachable,
 2592                                .max => switch (bits) {
 2593                                    else => unreachable,
 2594                                    16 => "__fmaxh",
 2595                                    32 => "fmaxf",
 2596                                    64 => "fmax",
 2597                                    80 => "__fmaxx",
 2598                                    128 => "fmaxq",
 2599                                },
 2600                                .min => switch (bits) {
 2601                                    else => unreachable,
 2602                                    16 => "__fminh",
 2603                                    32 => "fminf",
 2604                                    64 => "fmin",
 2605                                    80 => "__fminx",
 2606                                    128 => "fminq",
 2607                                },
 2608                            },
 2609                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 2610                        });
 2611                        try isel.emit(.bl(0));
 2612                        try call.finishCallee(isel);
 2613
 2614                        try call.prepareParams(isel);
 2615                        const lhs_vi = try isel.use(bin_op.lhs);
 2616                        const rhs_vi = try isel.use(bin_op.rhs);
 2617                        switch (bits) {
 2618                            else => unreachable,
 2619                            16, 32, 64, 128 => {
 2620                                try call.paramLiveOut(isel, rhs_vi, .v1);
 2621                                try call.paramLiveOut(isel, lhs_vi, .v0);
 2622                            },
 2623                            80 => {
 2624                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 2625                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 2626                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 2627                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 2628                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 2629                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 2630                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 2631                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 2632                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 2633                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 2634                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 2635                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 2636                            },
 2637                        }
 2638                        try call.finishParams(isel);
 2639                    },
 2640                }
 2641            }
 2642            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2643        },
 2644        .add_with_overflow, .sub_with_overflow => |air_tag| {
 2645            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
 2646                defer res_vi.value.deref(isel);
 2647
 2648                const ty_pl = air.data(air.inst_index).ty_pl;
 2649                const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
 2650                const ty = isel.air.typeOf(bin_op.lhs, ip);
 2651                const lhs_vi = try isel.use(bin_op.lhs);
 2652                const rhs_vi = try isel.use(bin_op.rhs);
 2653                const ty_size = lhs_vi.size(isel);
 2654                var overflow_it = res_vi.value.field(ty_pl.ty.toType(), ty_size, 1);
 2655                const overflow_vi = try overflow_it.only(isel);
 2656                var wrapped_it = res_vi.value.field(ty_pl.ty.toType(), 0, ty_size);
 2657                const wrapped_vi = try wrapped_it.only(isel);
 2658                try wrapped_vi.?.addOrSubtract(isel, ty, lhs_vi, switch (air_tag) {
 2659                    else => unreachable,
 2660                    .add_with_overflow => .add,
 2661                    .sub_with_overflow => .sub,
 2662                }, rhs_vi, .{
 2663                    .overflow = if (try overflow_vi.?.defReg(isel)) |overflow_ra| .{ .ra = overflow_ra } else .wrap,
 2664                });
 2665            }
 2666            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2667        },
 2668        .alloc, .ret_ptr => |air_tag| {
 2669            if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
 2670                defer ptr_vi.value.deref(isel);
 2671                switch (air_tag) {
 2672                    else => unreachable,
 2673                    .alloc => {},
 2674                    .ret_ptr => if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
 2675                        .unallocated, .stack_slot => {},
 2676                        .value, .constant => unreachable,
 2677                        .address => break :unused,
 2678                    },
 2679                }
 2680                const ptr_ra = try ptr_vi.value.defReg(isel) orelse break :unused;
 2681
 2682                const ty = air.data(air.inst_index).ty;
 2683                const slot_size = ty.childType(zcu).abiSize(zcu);
 2684                const slot_align = ty.ptrAlignment(zcu);
 2685                const slot_offset = slot_align.forward(isel.stack_size);
 2686                isel.stack_size = @intCast(slot_offset + slot_size);
 2687                const lo12: u12 = @truncate(slot_offset >> 0);
 2688                const hi12: u12 = @intCast(slot_offset >> 12);
 2689                if (hi12 > 0) try isel.emit(.add(
 2690                    ptr_ra.x(),
 2691                    if (lo12 > 0) ptr_ra.x() else .sp,
 2692                    .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 2693                ));
 2694                if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), .sp, .{ .immediate = lo12 }));
 2695            }
 2696            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2697        },
 2698        .inferred_alloc, .inferred_alloc_comptime => unreachable,
 2699        .assembly => {
 2700            const ty_pl = air.data(air.inst_index).ty_pl;
 2701            const extra = isel.air.extraData(Air.Asm, ty_pl.payload);
 2702            var extra_index = extra.end;
 2703            const outputs: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra_index..][0..extra.data.flags.outputs_len]);
 2704            extra_index += outputs.len;
 2705            const inputs: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra_index..][0..extra.data.inputs_len]);
 2706            extra_index += inputs.len;
 2707
 2708            var as: codegen.aarch64.Assemble = .{
 2709                .source = undefined,
 2710                .operands = .empty,
 2711            };
 2712            defer as.operands.deinit(gpa);
 2713
 2714            for (outputs) |output| {
 2715                const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
 2716                const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]), 0);
 2717                const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
 2718                // This equation accounts for the fact that even if we have exactly 4 bytes
 2719                // for the string, we still use the next u32 for the null terminator.
 2720                extra_index += (constraint.len + name.len + (2 + 3)) / 4;
 2721
 2722                switch (output) {
 2723                    else => return isel.fail("invalid constraint: '{s}'", .{constraint}),
 2724                    .none => if (std.mem.startsWith(u8, constraint, "={") and std.mem.endsWith(u8, constraint, "}")) {
 2725                        const output_reg = Register.parse(constraint["={".len .. constraint.len - "}".len]) orelse
 2726                            return isel.fail("invalid constraint: '{s}'", .{constraint});
 2727                        const output_ra = output_reg.alias;
 2728                        if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| {
 2729                            defer output_vi.value.deref(isel);
 2730                            try output_vi.value.defLiveIn(isel, output_reg.alias, comptime &.initFill(.free));
 2731                            isel.freeReg(output_ra);
 2732                        }
 2733                        if (!std.mem.eql(u8, name, "_")) {
 2734                            const operand_gop = try as.operands.getOrPut(gpa, name);
 2735                            if (operand_gop.found_existing) return isel.fail("duplicate output name: '{s}'", .{name});
 2736                            operand_gop.value_ptr.* = .{ .register = switch (ty_pl.ty.toType().abiSize(zcu)) {
 2737                                0 => unreachable,
 2738                                1...4 => output_ra.w(),
 2739                                5...8 => output_ra.x(),
 2740                                else => return isel.fail("too big output type: '{f}'", .{isel.fmtType(ty_pl.ty.toType())}),
 2741                            } };
 2742                        }
 2743                    } else if (std.mem.eql(u8, constraint, "=r")) {
 2744                        const output_ra = if (isel.live_values.fetchRemove(air.inst_index)) |output_vi| output_ra: {
 2745                            defer output_vi.value.deref(isel);
 2746                            break :output_ra try output_vi.value.defReg(isel) orelse try isel.allocIntReg();
 2747                        } else try isel.allocIntReg();
 2748                        if (!std.mem.eql(u8, name, "_")) {
 2749                            const operand_gop = try as.operands.getOrPut(gpa, name);
 2750                            if (operand_gop.found_existing) return isel.fail("duplicate output name: '{s}'", .{name});
 2751                            operand_gop.value_ptr.* = .{ .register = switch (ty_pl.ty.toType().abiSize(zcu)) {
 2752                                0 => unreachable,
 2753                                1...4 => output_ra.w(),
 2754                                5...8 => output_ra.x(),
 2755                                else => return isel.fail("too big output type: '{f}'", .{isel.fmtType(ty_pl.ty.toType())}),
 2756                            } };
 2757                        }
 2758                    } else return isel.fail("invalid constraint: '{s}'", .{constraint}),
 2759                }
 2760            }
 2761
 2762            const input_mats = try gpa.alloc(Value.Materialize, inputs.len);
 2763            defer gpa.free(input_mats);
 2764            const inputs_extra_index = extra_index;
 2765            for (inputs, input_mats) |input, *input_mat| {
 2766                const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
 2767                const constraint = std.mem.sliceTo(extra_bytes, 0);
 2768                const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
 2769                // This equation accounts for the fact that even if we have exactly 4 bytes
 2770                // for the string, we still use the next u32 for the null terminator.
 2771                extra_index += (constraint.len + name.len + (2 + 3)) / 4;
 2772
 2773                if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
 2774                    const input_reg = Register.parse(constraint["{".len .. constraint.len - "}".len]) orelse
 2775                        return isel.fail("invalid constraint: '{s}'", .{constraint});
 2776                    input_mat.* = .{ .vi = try isel.use(input), .ra = input_reg.alias };
 2777                    if (!std.mem.eql(u8, name, "_")) {
 2778                        const operand_gop = try as.operands.getOrPut(gpa, name);
 2779                        if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
 2780                        const input_ty = isel.air.typeOf(input, ip);
 2781                        operand_gop.value_ptr.* = .{ .register = switch (input_ty.abiSize(zcu)) {
 2782                            0 => unreachable,
 2783                            1...4 => input_reg.alias.w(),
 2784                            5...8 => input_reg.alias.x(),
 2785                            else => return isel.fail("too big input type: '{f}'", .{
 2786                                isel.fmtType(isel.air.typeOf(input, ip)),
 2787                            }),
 2788                        } };
 2789                    }
 2790                } else if (std.mem.eql(u8, constraint, "r")) {
 2791                    const input_vi = try isel.use(input);
 2792                    input_mat.* = try input_vi.matReg(isel);
 2793                    if (!std.mem.eql(u8, name, "_")) {
 2794                        const operand_gop = try as.operands.getOrPut(gpa, name);
 2795                        if (operand_gop.found_existing) return isel.fail("duplicate input name: '{s}'", .{name});
 2796                        operand_gop.value_ptr.* = .{ .register = switch (input_vi.size(isel)) {
 2797                            0 => unreachable,
 2798                            1...4 => input_mat.ra.w(),
 2799                            5...8 => input_mat.ra.x(),
 2800                            else => return isel.fail("too big input type: '{f}'", .{
 2801                                isel.fmtType(isel.air.typeOf(input, ip)),
 2802                            }),
 2803                        } };
 2804                    }
 2805                } else if (std.mem.eql(u8, name, "_")) {
 2806                    input_mat.vi = try isel.use(input);
 2807                } else return isel.fail("invalid constraint: '{s}'", .{constraint});
 2808            }
 2809
 2810            const clobbers = ip.indexToKey(extra.data.clobbers).aggregate;
 2811            const clobbers_ty: ZigType = .fromInterned(clobbers.ty);
 2812            for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
 2813                switch (switch (clobbers.storage) {
 2814                    .bytes => unreachable,
 2815                    .elems => |elems| elems[field_index],
 2816                    .repeated_elem => |repeated_elem| repeated_elem,
 2817                }) {
 2818                    else => unreachable,
 2819                    .bool_false => continue,
 2820                    .bool_true => {},
 2821                }
 2822                const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
 2823                if (std.mem.eql(u8, clobber_name, "memory")) continue;
 2824                if (std.mem.eql(u8, clobber_name, "nzcv")) continue;
 2825                const clobber_reg = Register.parse(clobber_name) orelse
 2826                    return isel.fail("unable to parse clobber: '{s}'", .{clobber_name});
 2827                const live_vi = isel.live_registers.getPtr(clobber_reg.alias);
 2828                switch (live_vi.*) {
 2829                    _ => {},
 2830                    .allocating => return isel.fail("clobbered twice: '{s}'", .{clobber_name}),
 2831                    .free => live_vi.* = .allocating,
 2832                }
 2833            }
 2834            for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
 2835                switch (switch (clobbers.storage) {
 2836                    .bytes => unreachable,
 2837                    .elems => |elems| elems[field_index],
 2838                    .repeated_elem => |repeated_elem| repeated_elem,
 2839                }) {
 2840                    else => unreachable,
 2841                    .bool_false => continue,
 2842                    .bool_true => {},
 2843                }
 2844                const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
 2845                if (std.mem.eql(u8, clobber_name, "memory")) continue;
 2846                if (std.mem.eql(u8, clobber_name, "nzcv")) continue;
 2847                const clobber_ra = Register.parse(clobber_name).?.alias;
 2848                const live_vi = isel.live_registers.getPtr(clobber_ra);
 2849                switch (live_vi.*) {
 2850                    _ => {
 2851                        if (!try isel.fill(clobber_ra))
 2852                            return isel.fail("unable to clobber: '{s}'", .{clobber_name});
 2853                        assert(live_vi.* == .free);
 2854                        live_vi.* = .allocating;
 2855                    },
 2856                    .allocating => {},
 2857                    .free => unreachable,
 2858                }
 2859            }
 2860
 2861            as.source = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..])[0..extra.data.source_len :0];
 2862            const asm_start = isel.instructions.items.len;
 2863            while (as.nextInstruction() catch |err| switch (err) {
 2864                error.InvalidSyntax => {
 2865                    const remaining_source = std.mem.span(as.source);
 2866                    return isel.fail("unable to assemble: '{s}'", .{std.mem.trim(
 2867                        u8,
 2868                        as.source[0 .. std.mem.indexOfScalar(u8, remaining_source, '\n') orelse remaining_source.len],
 2869                        &std.ascii.whitespace,
 2870                    )});
 2871                },
 2872            }) |instruction| try isel.emit(instruction);
 2873            std.mem.reverse(codegen.aarch64.encoding.Instruction, isel.instructions.items[asm_start..]);
 2874
 2875            extra_index = inputs_extra_index;
 2876            for (input_mats) |input_mat| {
 2877                const extra_bytes = std.mem.sliceAsBytes(isel.air.extra.items[extra_index..]);
 2878                const constraint = std.mem.sliceTo(extra_bytes, 0);
 2879                const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
 2880                // This equation accounts for the fact that even if we have exactly 4 bytes
 2881                // for the string, we still use the next u32 for the null terminator.
 2882                extra_index += (constraint.len + name.len + (2 + 3)) / 4;
 2883
 2884                if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) {
 2885                    try input_mat.vi.liveOut(isel, input_mat.ra);
 2886                } else if (std.mem.eql(u8, constraint, "r")) {
 2887                    try input_mat.finish(isel);
 2888                } else if (std.mem.eql(u8, name, "_")) {
 2889                    try input_mat.vi.mat(isel);
 2890                } else unreachable;
 2891            }
 2892
 2893            for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
 2894                switch (switch (clobbers.storage) {
 2895                    .bytes => unreachable,
 2896                    .elems => |elems| elems[field_index],
 2897                    .repeated_elem => |repeated_elem| repeated_elem,
 2898                }) {
 2899                    else => unreachable,
 2900                    .bool_false => continue,
 2901                    .bool_true => {},
 2902                }
 2903                const clobber_name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
 2904                if (std.mem.eql(u8, clobber_name, "memory")) continue;
 2905                if (std.mem.eql(u8, clobber_name, "cc")) continue;
 2906                isel.freeReg(Register.parse(clobber_name).?.alias);
 2907            }
 2908
 2909            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2910        },
 2911        .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| {
 2912            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
 2913                defer res_vi.value.deref(isel);
 2914
 2915                const bin_op = air.data(air.inst_index).bin_op;
 2916                const ty = isel.air.typeOf(bin_op.lhs, ip);
 2917                const int_info: std.builtin.Type.Int = if (ty.toIntern() == .bool_type)
 2918                    .{ .signedness = .unsigned, .bits = 1 }
 2919                else if (ty.isAbiInt(zcu))
 2920                    ty.intInfo(zcu)
 2921                else
 2922                    return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2923                if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2924
 2925                const lhs_vi = try isel.use(bin_op.lhs);
 2926                const rhs_vi = try isel.use(bin_op.rhs);
 2927                var offset = res_vi.value.size(isel);
 2928                while (offset > 0) {
 2929                    const size = @min(offset, 8);
 2930                    offset -= size;
 2931                    var res_part_it = res_vi.value.field(ty, offset, size);
 2932                    const res_part_vi = try res_part_it.only(isel);
 2933                    const res_part_ra = try res_part_vi.?.defReg(isel) orelse continue;
 2934                    var lhs_part_it = lhs_vi.field(ty, offset, size);
 2935                    const lhs_part_vi = try lhs_part_it.only(isel);
 2936                    const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
 2937                    var rhs_part_it = rhs_vi.field(ty, offset, size);
 2938                    const rhs_part_vi = try rhs_part_it.only(isel);
 2939                    const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
 2940                    try isel.emit(switch (air_tag) {
 2941                        else => unreachable,
 2942                        .bit_and, .bool_and => switch (size) {
 2943                            else => unreachable,
 2944                            1, 2, 4 => .@"and"(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 2945                            8 => .@"and"(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 2946                        },
 2947                        .bit_or, .bool_or => switch (size) {
 2948                            else => unreachable,
 2949                            1, 2, 4 => .orr(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 2950                            8 => .orr(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 2951                        },
 2952                        .xor => switch (size) {
 2953                            else => unreachable,
 2954                            1, 2, 4 => .eor(res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 2955                            8 => .eor(res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 2956                        },
 2957                    });
 2958                    try rhs_part_mat.finish(isel);
 2959                    try lhs_part_mat.finish(isel);
 2960                }
 2961            }
 2962            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 2963        },
 2964        .shr, .shr_exact, .shl, .shl_exact => |air_tag| {
 2965            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 2966                defer res_vi.value.deref(isel);
 2967
 2968                const bin_op = air.data(air.inst_index).bin_op;
 2969                const ty = isel.air.typeOf(bin_op.lhs, ip);
 2970                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 2971                const int_info = ty.intInfo(zcu);
 2972                switch (int_info.bits) {
 2973                    0 => unreachable,
 2974                    1...64 => |bits| {
 2975                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 2976                        switch (air_tag) {
 2977                            else => unreachable,
 2978                            .shr, .shr_exact, .shl_exact => {},
 2979                            .shl => switch (bits) {
 2980                                else => unreachable,
 2981                                1...31 => try isel.emit(switch (int_info.signedness) {
 2982                                    .signed => .sbfm(res_ra.w(), res_ra.w(), .{
 2983                                        .N = .word,
 2984                                        .immr = 0,
 2985                                        .imms = @intCast(bits - 1),
 2986                                    }),
 2987                                    .unsigned => .ubfm(res_ra.w(), res_ra.w(), .{
 2988                                        .N = .word,
 2989                                        .immr = 0,
 2990                                        .imms = @intCast(bits - 1),
 2991                                    }),
 2992                                }),
 2993                                32 => {},
 2994                                33...63 => try isel.emit(switch (int_info.signedness) {
 2995                                    .signed => .sbfm(res_ra.x(), res_ra.x(), .{
 2996                                        .N = .doubleword,
 2997                                        .immr = 0,
 2998                                        .imms = @intCast(bits - 1),
 2999                                    }),
 3000                                    .unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
 3001                                        .N = .doubleword,
 3002                                        .immr = 0,
 3003                                        .imms = @intCast(bits - 1),
 3004                                    }),
 3005                                }),
 3006                                64 => {},
 3007                            },
 3008                        }
 3009
 3010                        const lhs_vi = try isel.use(bin_op.lhs);
 3011                        const rhs_vi = try isel.use(bin_op.rhs);
 3012                        const lhs_mat = try lhs_vi.matReg(isel);
 3013                        const rhs_mat = try rhs_vi.matReg(isel);
 3014                        try isel.emit(switch (air_tag) {
 3015                            else => unreachable,
 3016                            .shr, .shr_exact => switch (bits) {
 3017                                else => unreachable,
 3018                                1...32 => switch (int_info.signedness) {
 3019                                    .signed => .asrv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 3020                                    .unsigned => .lsrv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 3021                                },
 3022                                33...64 => switch (int_info.signedness) {
 3023                                    .signed => .asrv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 3024                                    .unsigned => .lsrv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 3025                                },
 3026                            },
 3027                            .shl, .shl_exact => switch (bits) {
 3028                                else => unreachable,
 3029                                1...32 => .lslv(res_ra.w(), lhs_mat.ra.w(), rhs_mat.ra.w()),
 3030                                33...64 => .lslv(res_ra.x(), lhs_mat.ra.x(), rhs_mat.ra.x()),
 3031                            },
 3032                        });
 3033                        try rhs_mat.finish(isel);
 3034                        try lhs_mat.finish(isel);
 3035                    },
 3036                    65...128 => |bits| {
 3037                        var res_hi64_it = res_vi.value.field(ty, 8, 8);
 3038                        const res_hi64_vi = try res_hi64_it.only(isel);
 3039                        const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
 3040                        var res_lo64_it = res_vi.value.field(ty, 0, 8);
 3041                        const res_lo64_vi = try res_lo64_it.only(isel);
 3042                        const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
 3043                        if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
 3044                        if (res_hi64_ra) |res_ra| switch (air_tag) {
 3045                            else => unreachable,
 3046                            .shr, .shr_exact, .shl_exact => {},
 3047                            .shl => switch (bits) {
 3048                                else => unreachable,
 3049                                65...127 => try isel.emit(switch (int_info.signedness) {
 3050                                    .signed => .sbfm(res_ra.x(), res_ra.x(), .{
 3051                                        .N = .doubleword,
 3052                                        .immr = 0,
 3053                                        .imms = @intCast(bits - 64 - 1),
 3054                                    }),
 3055                                    .unsigned => .ubfm(res_ra.x(), res_ra.x(), .{
 3056                                        .N = .doubleword,
 3057                                        .immr = 0,
 3058                                        .imms = @intCast(bits - 64 - 1),
 3059                                    }),
 3060                                }),
 3061                                128 => {},
 3062                            },
 3063                        };
 3064
 3065                        const lhs_vi = try isel.use(bin_op.lhs);
 3066                        const lhs_hi64_mat = lhs_hi64_mat: {
 3067                            const res_lock: RegLock = switch (air_tag) {
 3068                                else => unreachable,
 3069                                .shr, .shr_exact => switch (int_info.signedness) {
 3070                                    .signed => if (res_lo64_ra) |res_ra| isel.lockReg(res_ra) else .empty,
 3071                                    .unsigned => .empty,
 3072                                },
 3073                                .shl, .shl_exact => .empty,
 3074                            };
 3075                            defer res_lock.unlock(isel);
 3076                            var lhs_hi64_it = lhs_vi.field(ty, 8, 8);
 3077                            const lhs_hi64_vi = try lhs_hi64_it.only(isel);
 3078                            break :lhs_hi64_mat try lhs_hi64_vi.?.matReg(isel);
 3079                        };
 3080                        var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 3081                        const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 3082                        const lhs_lo64_mat = try lhs_lo64_vi.?.matReg(isel);
 3083                        const rhs_vi = try isel.use(bin_op.rhs);
 3084                        const rhs_mat = try rhs_vi.matReg(isel);
 3085                        const lo64_ra = lo64_ra: {
 3086                            const res_lock: RegLock = switch (air_tag) {
 3087                                else => unreachable,
 3088                                .shr, .shr_exact => switch (int_info.signedness) {
 3089                                    .signed => if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
 3090                                    .unsigned => .empty,
 3091                                },
 3092                                .shl, .shl_exact => if (res_hi64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
 3093                            };
 3094                            defer res_lock.unlock(isel);
 3095                            break :lo64_ra try isel.allocIntReg();
 3096                        };
 3097                        defer isel.freeReg(lo64_ra);
 3098                        const hi64_ra = hi64_ra: {
 3099                            const res_lock: RegLock = switch (air_tag) {
 3100                                else => unreachable,
 3101                                .shr, .shr_exact => if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty,
 3102                                .shl, .shl_exact => .empty,
 3103                            };
 3104                            defer res_lock.unlock(isel);
 3105                            break :hi64_ra try isel.allocIntReg();
 3106                        };
 3107                        defer isel.freeReg(hi64_ra);
 3108                        switch (air_tag) {
 3109                            else => unreachable,
 3110                            .shr, .shr_exact => {
 3111                                if (res_hi64_ra) |res_ra| switch (int_info.signedness) {
 3112                                    .signed => {
 3113                                        try isel.emit(.csel(res_ra.x(), hi64_ra.x(), lo64_ra.x(), .eq));
 3114                                        try isel.emit(.sbfm(lo64_ra.x(), lhs_hi64_mat.ra.x(), .{
 3115                                            .N = .doubleword,
 3116                                            .immr = @intCast(bits - 64 - 1),
 3117                                            .imms = @intCast(bits - 64 - 1),
 3118                                        }));
 3119                                    },
 3120                                    .unsigned => try isel.emit(.csel(res_ra.x(), hi64_ra.x(), .xzr, .eq)),
 3121                                };
 3122                                if (res_lo64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), lo64_ra.x(), hi64_ra.x(), .eq));
 3123                                switch (int_info.signedness) {
 3124                                    .signed => try isel.emit(.asrv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x())),
 3125                                    .unsigned => try isel.emit(.lsrv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x())),
 3126                                }
 3127                            },
 3128                            .shl, .shl_exact => {
 3129                                if (res_lo64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), lo64_ra.x(), .xzr, .eq));
 3130                                if (res_hi64_ra) |res_ra| try isel.emit(.csel(res_ra.x(), hi64_ra.x(), lo64_ra.x(), .eq));
 3131                                try isel.emit(.lslv(lo64_ra.x(), lhs_lo64_mat.ra.x(), rhs_mat.ra.x()));
 3132                            },
 3133                        }
 3134                        try isel.emit(.ands(.wzr, rhs_mat.ra.w(), .{ .immediate = .{ .N = .word, .immr = 32 - 6, .imms = 0 } }));
 3135                        switch (air_tag) {
 3136                            else => unreachable,
 3137                            .shr, .shr_exact => if (res_lo64_ra) |_| {
 3138                                try isel.emit(.orr(
 3139                                    lo64_ra.x(),
 3140                                    lo64_ra.x(),
 3141                                    .{ .shifted_register = .{ .register = hi64_ra.x(), .shift = .{ .lsl = 1 } } },
 3142                                ));
 3143                                try isel.emit(.lslv(hi64_ra.x(), lhs_hi64_mat.ra.x(), hi64_ra.x()));
 3144                                try isel.emit(.lsrv(lo64_ra.x(), lhs_lo64_mat.ra.x(), rhs_mat.ra.x()));
 3145                                try isel.emit(.orn(hi64_ra.w(), .wzr, .{ .register = rhs_mat.ra.w() }));
 3146                            },
 3147                            .shl, .shl_exact => if (res_hi64_ra) |_| {
 3148                                try isel.emit(.orr(
 3149                                    hi64_ra.x(),
 3150                                    hi64_ra.x(),
 3151                                    .{ .shifted_register = .{ .register = lo64_ra.x(), .shift = .{ .lsr = 1 } } },
 3152                                ));
 3153                                try isel.emit(.lsrv(lo64_ra.x(), lhs_lo64_mat.ra.x(), lo64_ra.x()));
 3154                                try isel.emit(.lslv(hi64_ra.x(), lhs_hi64_mat.ra.x(), rhs_mat.ra.x()));
 3155                                try isel.emit(.orn(lo64_ra.w(), .wzr, .{ .register = rhs_mat.ra.w() }));
 3156                            },
 3157                        }
 3158                        try rhs_mat.finish(isel);
 3159                        try lhs_lo64_mat.finish(isel);
 3160                        try lhs_hi64_mat.finish(isel);
 3161                        break :unused;
 3162                    },
 3163                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 3164                }
 3165            }
 3166            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3167        },
 3168        .not => |air_tag| {
 3169            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
 3170                defer res_vi.value.deref(isel);
 3171
 3172                const ty_op = air.data(air.inst_index).ty_op;
 3173                const ty = ty_op.ty.toType();
 3174                const int_info: std.builtin.Type.Int = int_info: {
 3175                    if (ty_op.ty == .bool_type) break :int_info .{ .signedness = .unsigned, .bits = 1 };
 3176                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3177                    break :int_info ty.intInfo(zcu);
 3178                };
 3179                if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3180
 3181                const src_vi = try isel.use(ty_op.operand);
 3182                var offset = res_vi.value.size(isel);
 3183                while (offset > 0) {
 3184                    const size = @min(offset, 8);
 3185                    offset -= size;
 3186                    var res_part_it = res_vi.value.field(ty, offset, size);
 3187                    const res_part_vi = try res_part_it.only(isel);
 3188                    const res_part_ra = try res_part_vi.?.defReg(isel) orelse continue;
 3189                    var src_part_it = src_vi.field(ty, offset, size);
 3190                    const src_part_vi = try src_part_it.only(isel);
 3191                    const src_part_mat = try src_part_vi.?.matReg(isel);
 3192                    try isel.emit(switch (int_info.signedness) {
 3193                        .signed => switch (size) {
 3194                            else => unreachable,
 3195                            1, 2, 4 => .orn(res_part_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
 3196                            8 => .orn(res_part_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
 3197                        },
 3198                        .unsigned => switch (@min(int_info.bits - 8 * offset, 64)) {
 3199                            else => unreachable,
 3200                            1...31 => |bits| .eor(res_part_ra.w(), src_part_mat.ra.w(), .{ .immediate = .{
 3201                                .N = .word,
 3202                                .immr = 0,
 3203                                .imms = @intCast(bits - 1),
 3204                            } }),
 3205                            32 => .orn(res_part_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
 3206                            33...63 => |bits| .eor(res_part_ra.x(), src_part_mat.ra.x(), .{ .immediate = .{
 3207                                .N = .doubleword,
 3208                                .immr = 0,
 3209                                .imms = @intCast(bits - 1),
 3210                            } }),
 3211                            64 => .orn(res_part_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
 3212                        },
 3213                    });
 3214                    try src_part_mat.finish(isel);
 3215                }
 3216            }
 3217            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3218        },
 3219        .bitcast => |air_tag| {
 3220            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 3221                defer dst_vi.value.deref(isel);
 3222                const ty_op = air.data(air.inst_index).ty_op;
 3223                const dst_ty = ty_op.ty.toType();
 3224                const dst_tag = dst_ty.zigTypeTag(zcu);
 3225                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 3226                const src_tag = src_ty.zigTypeTag(zcu);
 3227                if (dst_ty.isAbiInt(zcu) and (src_tag == .bool or src_ty.isAbiInt(zcu))) {
 3228                    const dst_int_info = dst_ty.intInfo(zcu);
 3229                    const src_int_info: std.builtin.Type.Int = if (src_tag == .bool) .{ .signedness = undefined, .bits = 1 } else src_ty.intInfo(zcu);
 3230                    assert(dst_int_info.bits == src_int_info.bits);
 3231                    if (dst_tag != .@"struct" and src_tag != .@"struct" and src_tag != .bool and dst_int_info.signedness == src_int_info.signedness) {
 3232                        try dst_vi.value.move(isel, ty_op.operand);
 3233                    } else switch (dst_int_info.bits) {
 3234                        0 => unreachable,
 3235                        1...31 => |dst_bits| {
 3236                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3237                            const src_vi = try isel.use(ty_op.operand);
 3238                            const src_mat = try src_vi.matReg(isel);
 3239                            try isel.emit(switch (dst_int_info.signedness) {
 3240                                .signed => .sbfm(dst_ra.w(), src_mat.ra.w(), .{
 3241                                    .N = .word,
 3242                                    .immr = 0,
 3243                                    .imms = @intCast(dst_bits - 1),
 3244                                }),
 3245                                .unsigned => .ubfm(dst_ra.w(), src_mat.ra.w(), .{
 3246                                    .N = .word,
 3247                                    .immr = 0,
 3248                                    .imms = @intCast(dst_bits - 1),
 3249                                }),
 3250                            });
 3251                            try src_mat.finish(isel);
 3252                        },
 3253                        32 => try dst_vi.value.move(isel, ty_op.operand),
 3254                        33...63 => |dst_bits| {
 3255                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3256                            const src_vi = try isel.use(ty_op.operand);
 3257                            const src_mat = try src_vi.matReg(isel);
 3258                            try isel.emit(switch (dst_int_info.signedness) {
 3259                                .signed => .sbfm(dst_ra.x(), src_mat.ra.x(), .{
 3260                                    .N = .doubleword,
 3261                                    .immr = 0,
 3262                                    .imms = @intCast(dst_bits - 1),
 3263                                }),
 3264                                .unsigned => .ubfm(dst_ra.x(), src_mat.ra.x(), .{
 3265                                    .N = .doubleword,
 3266                                    .immr = 0,
 3267                                    .imms = @intCast(dst_bits - 1),
 3268                                }),
 3269                            });
 3270                            try src_mat.finish(isel);
 3271                        },
 3272                        64 => try dst_vi.value.move(isel, ty_op.operand),
 3273                        65...127 => |dst_bits| {
 3274                            const src_vi = try isel.use(ty_op.operand);
 3275                            var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
 3276                            const dst_hi64_vi = try dst_hi64_it.only(isel);
 3277                            if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| {
 3278                                var src_hi64_it = src_vi.field(src_ty, 8, 8);
 3279                                const src_hi64_vi = try src_hi64_it.only(isel);
 3280                                const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 3281                                try isel.emit(switch (dst_int_info.signedness) {
 3282                                    .signed => .sbfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
 3283                                        .N = .doubleword,
 3284                                        .immr = 0,
 3285                                        .imms = @intCast(dst_bits - 64 - 1),
 3286                                    }),
 3287                                    .unsigned => .ubfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
 3288                                        .N = .doubleword,
 3289                                        .immr = 0,
 3290                                        .imms = @intCast(dst_bits - 64 - 1),
 3291                                    }),
 3292                                });
 3293                                try src_hi64_mat.finish(isel);
 3294                            }
 3295                            var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 3296                            const dst_lo64_vi = try dst_lo64_it.only(isel);
 3297                            if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
 3298                                var src_lo64_it = src_vi.field(src_ty, 0, 8);
 3299                                const src_lo64_vi = try src_lo64_it.only(isel);
 3300                                try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
 3301                            }
 3302                        },
 3303                        128 => try dst_vi.value.move(isel, ty_op.operand),
 3304                        else => return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
 3305                    }
 3306                } else if ((dst_ty.isPtrAtRuntime(zcu) or dst_ty.isAbiInt(zcu)) and (src_ty.isPtrAtRuntime(zcu) or src_ty.isAbiInt(zcu))) {
 3307                    try dst_vi.value.move(isel, ty_op.operand);
 3308                } else if (dst_ty.isSliceAtRuntime(zcu) and src_ty.isSliceAtRuntime(zcu)) {
 3309                    try dst_vi.value.move(isel, ty_op.operand);
 3310                } else if (dst_tag == .error_union and src_tag == .error_union) {
 3311                    assert(dst_ty.errorUnionSet(zcu).hasRuntimeBitsIgnoreComptime(zcu) ==
 3312                        src_ty.errorUnionSet(zcu).hasRuntimeBitsIgnoreComptime(zcu));
 3313                    if (dst_ty.errorUnionPayload(zcu).toIntern() == src_ty.errorUnionPayload(zcu).toIntern()) {
 3314                        try dst_vi.value.move(isel, ty_op.operand);
 3315                    } else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 3316                } else if (dst_tag == .float and src_tag == .float) {
 3317                    assert(dst_ty.floatBits(isel.target) == src_ty.floatBits(isel.target));
 3318                    try dst_vi.value.move(isel, ty_op.operand);
 3319                } else if (dst_ty.isAbiInt(zcu) and src_tag == .float) {
 3320                    const dst_int_info = dst_ty.intInfo(zcu);
 3321                    assert(dst_int_info.bits == src_ty.floatBits(isel.target));
 3322                    switch (dst_int_info.bits) {
 3323                        else => unreachable,
 3324                        16 => {
 3325                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3326                            const src_vi = try isel.use(ty_op.operand);
 3327                            const src_mat = try src_vi.matReg(isel);
 3328                            switch (dst_int_info.signedness) {
 3329                                .signed => try isel.emit(.smov(dst_ra.w(), src_mat.ra.@"h[]"(0))),
 3330                                .unsigned => try isel.emit(if (isel.target.cpu.has(.aarch64, .fullfp16))
 3331                                    .fmov(dst_ra.w(), .{ .register = src_mat.ra.h() })
 3332                                else
 3333                                    .umov(dst_ra.w(), src_mat.ra.@"h[]"(0))),
 3334                            }
 3335                            try src_mat.finish(isel);
 3336                        },
 3337                        32 => {
 3338                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3339                            const src_vi = try isel.use(ty_op.operand);
 3340                            const src_mat = try src_vi.matReg(isel);
 3341                            try isel.emit(.fmov(dst_ra.w(), .{ .register = src_mat.ra.s() }));
 3342                            try src_mat.finish(isel);
 3343                        },
 3344                        64 => {
 3345                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3346                            const src_vi = try isel.use(ty_op.operand);
 3347                            const src_mat = try src_vi.matReg(isel);
 3348                            try isel.emit(.fmov(dst_ra.x(), .{ .register = src_mat.ra.d() }));
 3349                            try src_mat.finish(isel);
 3350                        },
 3351                        80 => switch (dst_int_info.signedness) {
 3352                            .signed => {
 3353                                const src_vi = try isel.use(ty_op.operand);
 3354                                var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
 3355                                const dst_hi16_vi = try dst_hi16_it.only(isel);
 3356                                if (try dst_hi16_vi.?.defReg(isel)) |dst_hi16_ra| {
 3357                                    var src_hi16_it = src_vi.field(src_ty, 8, 8);
 3358                                    const src_hi16_vi = try src_hi16_it.only(isel);
 3359                                    const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
 3360                                    try isel.emit(.sbfm(dst_hi16_ra.x(), src_hi16_mat.ra.x(), .{
 3361                                        .N = .doubleword,
 3362                                        .immr = 0,
 3363                                        .imms = 16 - 1,
 3364                                    }));
 3365                                    try src_hi16_mat.finish(isel);
 3366                                }
 3367                                var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 3368                                const dst_lo64_vi = try dst_lo64_it.only(isel);
 3369                                if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
 3370                                    var src_lo64_it = src_vi.field(src_ty, 0, 8);
 3371                                    const src_lo64_vi = try src_lo64_it.only(isel);
 3372                                    try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
 3373                                }
 3374                            },
 3375                            else => try dst_vi.value.move(isel, ty_op.operand),
 3376                        },
 3377                        128 => {
 3378                            const src_vi = try isel.use(ty_op.operand);
 3379                            const src_mat = try src_vi.matReg(isel);
 3380                            var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
 3381                            const dst_hi64_vi = try dst_hi64_it.only(isel);
 3382                            if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| try isel.emit(.fmov(dst_hi64_ra.x(), .{ .register = src_mat.ra.@"d[]"(1) }));
 3383                            var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 3384                            const dst_lo64_vi = try dst_lo64_it.only(isel);
 3385                            if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| try isel.emit(.fmov(dst_lo64_ra.x(), .{ .register = src_mat.ra.d() }));
 3386                            try src_mat.finish(isel);
 3387                        },
 3388                    }
 3389                } else if (dst_tag == .float and src_ty.isAbiInt(zcu)) {
 3390                    const src_int_info = src_ty.intInfo(zcu);
 3391                    assert(dst_ty.floatBits(isel.target) == src_int_info.bits);
 3392                    switch (src_int_info.bits) {
 3393                        else => unreachable,
 3394                        16 => {
 3395                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3396                            const src_vi = try isel.use(ty_op.operand);
 3397                            const src_mat = try src_vi.matReg(isel);
 3398                            try isel.emit(.fmov(
 3399                                if (isel.target.cpu.has(.aarch64, .fullfp16)) dst_ra.h() else dst_ra.s(),
 3400                                .{ .register = src_mat.ra.w() },
 3401                            ));
 3402                            try src_mat.finish(isel);
 3403                        },
 3404                        32 => {
 3405                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3406                            const src_vi = try isel.use(ty_op.operand);
 3407                            const src_mat = try src_vi.matReg(isel);
 3408                            try isel.emit(.fmov(dst_ra.s(), .{ .register = src_mat.ra.w() }));
 3409                            try src_mat.finish(isel);
 3410                        },
 3411                        64 => {
 3412                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3413                            const src_vi = try isel.use(ty_op.operand);
 3414                            const src_mat = try src_vi.matReg(isel);
 3415                            try isel.emit(.fmov(dst_ra.d(), .{ .register = src_mat.ra.x() }));
 3416                            try src_mat.finish(isel);
 3417                        },
 3418                        80 => switch (src_int_info.signedness) {
 3419                            .signed => {
 3420                                const src_vi = try isel.use(ty_op.operand);
 3421                                var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
 3422                                const dst_hi16_vi = try dst_hi16_it.only(isel);
 3423                                if (try dst_hi16_vi.?.defReg(isel)) |dst_hi16_ra| {
 3424                                    var src_hi16_it = src_vi.field(src_ty, 8, 8);
 3425                                    const src_hi16_vi = try src_hi16_it.only(isel);
 3426                                    const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
 3427                                    try isel.emit(.ubfm(dst_hi16_ra.x(), src_hi16_mat.ra.x(), .{
 3428                                        .N = .doubleword,
 3429                                        .immr = 0,
 3430                                        .imms = 16 - 1,
 3431                                    }));
 3432                                    try src_hi16_mat.finish(isel);
 3433                                }
 3434                                var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 3435                                const dst_lo64_vi = try dst_lo64_it.only(isel);
 3436                                if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
 3437                                    var src_lo64_it = src_vi.field(src_ty, 0, 8);
 3438                                    const src_lo64_vi = try src_lo64_it.only(isel);
 3439                                    try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
 3440                                }
 3441                            },
 3442                            else => try dst_vi.value.move(isel, ty_op.operand),
 3443                        },
 3444                        128 => {
 3445                            const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 3446                            const src_vi = try isel.use(ty_op.operand);
 3447                            var src_hi64_it = src_vi.field(src_ty, 8, 8);
 3448                            const src_hi64_vi = try src_hi64_it.only(isel);
 3449                            const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 3450                            try isel.emit(.fmov(dst_ra.@"d[]"(1), .{ .register = src_hi64_mat.ra.x() }));
 3451                            try src_hi64_mat.finish(isel);
 3452                            var src_lo64_it = src_vi.field(src_ty, 0, 8);
 3453                            const src_lo64_vi = try src_lo64_it.only(isel);
 3454                            const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
 3455                            try isel.emit(.fmov(dst_ra.d(), .{ .register = src_lo64_mat.ra.x() }));
 3456                            try src_lo64_mat.finish(isel);
 3457                        },
 3458                    }
 3459                } else if (dst_ty.isAbiInt(zcu) and src_tag == .array and src_ty.childType(zcu).isAbiInt(zcu)) {
 3460                    const dst_int_info = dst_ty.intInfo(zcu);
 3461                    const src_child_int_info = src_ty.childType(zcu).intInfo(zcu);
 3462                    const src_len = src_ty.arrayLenIncludingSentinel(zcu);
 3463                    assert(dst_int_info.bits == src_child_int_info.bits * src_len);
 3464                    const src_child_size = src_ty.childType(zcu).abiSize(zcu);
 3465                    if (8 * src_child_size == src_child_int_info.bits) {
 3466                        try dst_vi.value.defAddr(isel, dst_ty, .{ .wrap = dst_int_info }) orelse break :unused;
 3467
 3468                        try call.prepareReturn(isel);
 3469                        try call.finishReturn(isel);
 3470
 3471                        try call.prepareCallee(isel);
 3472                        try isel.global_relocs.append(gpa, .{
 3473                            .name = "memcpy",
 3474                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 3475                        });
 3476                        try isel.emit(.bl(0));
 3477                        try call.finishCallee(isel);
 3478
 3479                        try call.prepareParams(isel);
 3480                        const src_vi = try isel.use(ty_op.operand);
 3481                        try isel.movImmediate(.x2, src_child_size * src_len);
 3482                        try call.paramAddress(isel, src_vi, .r1);
 3483                        try call.paramAddress(isel, dst_vi.value, .r0);
 3484                        try call.finishParams(isel);
 3485                    } else return isel.fail("bad  {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 3486                } else if (dst_tag == .array and dst_ty.childType(zcu).isAbiInt(zcu) and src_ty.isAbiInt(zcu)) {
 3487                    const dst_child_int_info = dst_ty.childType(zcu).intInfo(zcu);
 3488                    const src_int_info = src_ty.intInfo(zcu);
 3489                    const dst_len = dst_ty.arrayLenIncludingSentinel(zcu);
 3490                    assert(dst_child_int_info.bits * dst_len == src_int_info.bits);
 3491                    const dst_child_size = dst_ty.childType(zcu).abiSize(zcu);
 3492                    if (8 * dst_child_size == dst_child_int_info.bits) {
 3493                        try dst_vi.value.defAddr(isel, dst_ty, .{}) orelse break :unused;
 3494
 3495                        try call.prepareReturn(isel);
 3496                        try call.finishReturn(isel);
 3497
 3498                        try call.prepareCallee(isel);
 3499                        try isel.global_relocs.append(gpa, .{
 3500                            .name = "memcpy",
 3501                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 3502                        });
 3503                        try isel.emit(.bl(0));
 3504                        try call.finishCallee(isel);
 3505
 3506                        try call.prepareParams(isel);
 3507                        const src_vi = try isel.use(ty_op.operand);
 3508                        try isel.movImmediate(.x2, dst_child_size * dst_len);
 3509                        try call.paramAddress(isel, src_vi, .r1);
 3510                        try call.paramAddress(isel, dst_vi.value, .r0);
 3511                        try call.finishParams(isel);
 3512                    } else return isel.fail("bad  {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 3513                } else return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 3514            }
 3515            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3516        },
 3517        .block => {
 3518            const ty_pl = air.data(air.inst_index).ty_pl;
 3519            const extra = isel.air.extraData(Air.Block, ty_pl.payload);
 3520            try isel.block(air.inst_index, ty_pl.ty.toType(), @ptrCast(
 3521                isel.air.extra.items[extra.end..][0..extra.data.body_len],
 3522            ));
 3523            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3524        },
 3525        .loop => {
 3526            const ty_pl = air.data(air.inst_index).ty_pl;
 3527            const extra = isel.air.extraData(Air.Block, ty_pl.payload);
 3528            const loops = isel.loops.values();
 3529            const loop_index = isel.loops.getIndex(air.inst_index).?;
 3530            const loop = &loops[loop_index];
 3531
 3532            tracking_log.debug("{f}", .{
 3533                isel.fmtDom(air.inst_index, loop.dom, @intCast(isel.blocks.count())),
 3534            });
 3535            tracking_log.debug("{f}", .{isel.fmtLoopLive(air.inst_index)});
 3536            assert(loop.depth == isel.blocks.count());
 3537
 3538            if (false) {
 3539                // loops are dumb...
 3540                for (isel.loop_live.list.items[loop.live..loops[loop_index + 1].live]) |live_inst| {
 3541                    const live_vi = try isel.use(live_inst.toRef());
 3542                    try live_vi.mat(isel);
 3543                }
 3544
 3545                // IT'S DOM TIME!!!
 3546                for (isel.blocks.values(), 0..) |*dom_block, dom_index| {
 3547                    if (@as(u1, @truncate(isel.dom.items[
 3548                        loop.dom + dom_index / @bitSizeOf(DomInt)
 3549                    ] >> @truncate(dom_index))) == 0) continue;
 3550                    var live_reg_it = dom_block.live_registers.iterator();
 3551                    while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
 3552                        _ => |live_vi| try live_vi.mat(isel),
 3553                        .allocating => unreachable,
 3554                        .free => {},
 3555                    };
 3556                }
 3557            }
 3558
 3559            loop.live_registers = isel.live_registers;
 3560            loop.repeat_list = Loop.empty_list;
 3561            try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
 3562            try isel.merge(&loop.live_registers, .{ .fill_extra = true });
 3563
 3564            var repeat_label = loop.repeat_list;
 3565            assert(repeat_label != Loop.empty_list);
 3566            while (repeat_label != Loop.empty_list) {
 3567                const instruction = &isel.instructions.items[repeat_label];
 3568                const next_repeat_label = instruction.*;
 3569                instruction.* = .b(-@as(i28, @intCast((isel.instructions.items.len - 1 - repeat_label) << 2)));
 3570                repeat_label = @bitCast(next_repeat_label);
 3571            }
 3572
 3573            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3574        },
 3575        .repeat => {
 3576            const repeat = air.data(air.inst_index).repeat;
 3577            try isel.loops.getPtr(repeat.loop_inst).?.branch(isel);
 3578            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3579        },
 3580        .br => {
 3581            const br = air.data(air.inst_index).br;
 3582            try isel.blocks.getPtr(br.block_inst).?.branch(isel);
 3583            if (isel.live_values.get(br.block_inst)) |dst_vi| try dst_vi.move(isel, br.operand);
 3584            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3585        },
 3586        .trap => {
 3587            try isel.emit(.brk(0x1));
 3588            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3589        },
 3590        .breakpoint => {
 3591            try isel.emit(.brk(0xf000));
 3592            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3593        },
 3594        .ret_addr => {
 3595            if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
 3596                defer addr_vi.value.deref(isel);
 3597                const addr_ra = try addr_vi.value.defReg(isel) orelse break :unused;
 3598                try isel.emit(.ldr(addr_ra.x(), .{ .unsigned_offset = .{ .base = .fp, .offset = 8 } }));
 3599            }
 3600            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3601        },
 3602        .frame_addr => {
 3603            if (isel.live_values.fetchRemove(air.inst_index)) |addr_vi| unused: {
 3604                defer addr_vi.value.deref(isel);
 3605                const addr_ra = try addr_vi.value.defReg(isel) orelse break :unused;
 3606                try isel.emit(.orr(addr_ra.x(), .xzr, .{ .register = .fp }));
 3607            }
 3608            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3609        },
 3610        .call => {
 3611            const pl_op = air.data(air.inst_index).pl_op;
 3612            const extra = isel.air.extraData(Air.Call, pl_op.payload);
 3613            const args: []const Air.Inst.Ref = @ptrCast(isel.air.extra.items[extra.end..][0..extra.data.args_len]);
 3614            const callee_ty = isel.air.typeOf(pl_op.operand, ip);
 3615            const func_info = switch (ip.indexToKey(callee_ty.toIntern())) {
 3616                else => unreachable,
 3617                .func_type => |func_type| func_type,
 3618                .ptr_type => |ptr_type| ip.indexToKey(ptr_type.child).func_type,
 3619            };
 3620
 3621            try call.prepareReturn(isel);
 3622            const maybe_def_ret_vi = isel.live_values.fetchRemove(air.inst_index);
 3623            var maybe_ret_addr_vi: ?Value.Index = null;
 3624            if (maybe_def_ret_vi) |def_ret_vi| {
 3625                defer def_ret_vi.value.deref(isel);
 3626
 3627                var ret_it: CallAbiIterator = .init;
 3628                const ret_vi = try ret_it.ret(isel, isel.air.typeOfIndex(air.inst_index, ip));
 3629                defer ret_vi.?.deref(isel);
 3630                switch (ret_vi.?.parent(isel)) {
 3631                    .unallocated, .stack_slot => if (ret_vi.?.hint(isel)) |ret_ra| {
 3632                        try call.returnLiveIn(isel, def_ret_vi.value, ret_ra);
 3633                    } else {
 3634                        var def_ret_part_it = def_ret_vi.value.parts(isel);
 3635                        var ret_part_it = ret_vi.?.parts(isel);
 3636                        while (def_ret_part_it.next()) |ret_part_vi| {
 3637                            try call.returnLiveIn(isel, ret_part_vi, ret_part_it.next().?.hint(isel).?);
 3638                        }
 3639                    },
 3640                    .value, .constant => unreachable,
 3641                    .address => |address_vi| {
 3642                        maybe_ret_addr_vi = address_vi;
 3643                        _ = try def_ret_vi.value.defAddr(isel, isel.air.typeOfIndex(air.inst_index, ip), .{
 3644                            .expected_live_registers = &call.caller_saved_regs,
 3645                        });
 3646                    },
 3647                }
 3648            }
 3649            try call.finishReturn(isel);
 3650
 3651            try call.prepareCallee(isel);
 3652            if (pl_op.operand.toInterned()) |ct_callee| {
 3653                try isel.nav_relocs.append(gpa, switch (ip.indexToKey(ct_callee)) {
 3654                    else => unreachable,
 3655                    inline .@"extern", .func => |func| .{
 3656                        .nav = func.owner_nav,
 3657                        .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 3658                    },
 3659                    .ptr => |ptr| .{
 3660                        .nav = ptr.base_addr.nav,
 3661                        .reloc = .{
 3662                            .label = @intCast(isel.instructions.items.len),
 3663                            .addend = ptr.byte_offset,
 3664                        },
 3665                    },
 3666                });
 3667                try isel.emit(.bl(0));
 3668            } else {
 3669                const callee_vi = try isel.use(pl_op.operand);
 3670                const callee_mat = try callee_vi.matReg(isel);
 3671                try isel.emit(.blr(callee_mat.ra.x()));
 3672                try callee_mat.finish(isel);
 3673            }
 3674            try call.finishCallee(isel);
 3675
 3676            try call.prepareParams(isel);
 3677            if (maybe_ret_addr_vi) |ret_addr_vi| try call.paramAddress(
 3678                isel,
 3679                maybe_def_ret_vi.?.value,
 3680                ret_addr_vi.hint(isel).?,
 3681            );
 3682            var param_it: CallAbiIterator = .init;
 3683            for (args, 0..) |arg, arg_index| {
 3684                const param_ty = isel.air.typeOf(arg, ip);
 3685                const param_vi = param_vi: {
 3686                    if (arg_index >= func_info.param_types.len) {
 3687                        assert(func_info.is_var_args);
 3688                        switch (isel.va_list) {
 3689                            .other => break :param_vi try param_it.nonSysvVarArg(isel, param_ty),
 3690                            .sysv => {},
 3691                        }
 3692                    }
 3693                    break :param_vi try param_it.param(isel, param_ty);
 3694                } orelse continue;
 3695                defer param_vi.deref(isel);
 3696                const arg_vi = try isel.use(arg);
 3697                switch (param_vi.parent(isel)) {
 3698                    .unallocated => if (param_vi.hint(isel)) |param_ra| {
 3699                        try call.paramLiveOut(isel, arg_vi, param_ra);
 3700                    } else {
 3701                        var param_part_it = param_vi.parts(isel);
 3702                        var arg_part_it = arg_vi.parts(isel);
 3703                        if (arg_part_it.only()) |_| {
 3704                            try isel.values.ensureUnusedCapacity(gpa, param_part_it.remaining);
 3705                            arg_vi.setParts(isel, param_part_it.remaining);
 3706                            while (param_part_it.next()) |param_part_vi| _ = arg_vi.addPart(
 3707                                isel,
 3708                                param_part_vi.get(isel).offset_from_parent,
 3709                                param_part_vi.size(isel),
 3710                            );
 3711                            param_part_it = param_vi.parts(isel);
 3712                            arg_part_it = arg_vi.parts(isel);
 3713                        }
 3714                        while (param_part_it.next()) |param_part_vi| {
 3715                            const arg_part_vi = arg_part_it.next().?;
 3716                            assert(arg_part_vi.get(isel).offset_from_parent ==
 3717                                param_part_vi.get(isel).offset_from_parent);
 3718                            assert(arg_part_vi.size(isel) == param_part_vi.size(isel));
 3719                            try call.paramLiveOut(isel, arg_part_vi, param_part_vi.hint(isel).?);
 3720                        }
 3721                    },
 3722                    .stack_slot => |stack_slot| try arg_vi.store(isel, param_ty, stack_slot.base, .{
 3723                        .offset = @intCast(stack_slot.offset),
 3724                    }),
 3725                    .value, .constant => unreachable,
 3726                    .address => |address_vi| try call.paramAddress(isel, arg_vi, address_vi.hint(isel).?),
 3727                }
 3728            }
 3729            try call.finishParams(isel);
 3730
 3731            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3732        },
 3733        .clz => |air_tag| {
 3734            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3735                defer res_vi.value.deref(isel);
 3736
 3737                const ty_op = air.data(air.inst_index).ty_op;
 3738                const ty = isel.air.typeOf(ty_op.operand, ip);
 3739                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3740                const int_info = ty.intInfo(zcu);
 3741                switch (int_info.bits) {
 3742                    0 => unreachable,
 3743                    1...64 => {
 3744                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3745                        const src_vi = try isel.use(ty_op.operand);
 3746                        const src_mat = try src_vi.matReg(isel);
 3747                        try isel.clzLimb(res_ra, int_info, src_mat.ra);
 3748                        try src_mat.finish(isel);
 3749                    },
 3750                    65...128 => |bits| {
 3751                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3752                        const src_vi = try isel.use(ty_op.operand);
 3753                        var src_hi64_it = src_vi.field(ty, 8, 8);
 3754                        const src_hi64_vi = try src_hi64_it.only(isel);
 3755                        const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 3756                        var src_lo64_it = src_vi.field(ty, 0, 8);
 3757                        const src_lo64_vi = try src_lo64_it.only(isel);
 3758                        const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
 3759                        const lo64_ra = try isel.allocIntReg();
 3760                        defer isel.freeReg(lo64_ra);
 3761                        const hi64_ra = try isel.allocIntReg();
 3762                        defer isel.freeReg(hi64_ra);
 3763                        try isel.emit(.csel(res_ra.w(), lo64_ra.w(), hi64_ra.w(), .eq));
 3764                        try isel.emit(.add(lo64_ra.w(), lo64_ra.w(), .{ .immediate = @intCast(bits - 64) }));
 3765                        try isel.emit(.subs(.xzr, src_hi64_mat.ra.x(), .{ .immediate = 0 }));
 3766                        try isel.clzLimb(hi64_ra, .{ .signedness = int_info.signedness, .bits = bits - 64 }, src_hi64_mat.ra);
 3767                        try isel.clzLimb(lo64_ra, .{ .signedness = .unsigned, .bits = 64 }, src_lo64_mat.ra);
 3768                        try src_hi64_mat.finish(isel);
 3769                        try src_lo64_mat.finish(isel);
 3770                    },
 3771                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 3772                }
 3773            }
 3774            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3775        },
 3776        .ctz => |air_tag| {
 3777            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3778                defer res_vi.value.deref(isel);
 3779
 3780                const ty_op = air.data(air.inst_index).ty_op;
 3781                const ty = isel.air.typeOf(ty_op.operand, ip);
 3782                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3783                const int_info = ty.intInfo(zcu);
 3784                switch (int_info.bits) {
 3785                    0 => unreachable,
 3786                    1...64 => {
 3787                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3788                        const src_vi = try isel.use(ty_op.operand);
 3789                        const src_mat = try src_vi.matReg(isel);
 3790                        try isel.ctzLimb(res_ra, int_info, src_mat.ra);
 3791                        try src_mat.finish(isel);
 3792                    },
 3793                    65...128 => |bits| {
 3794                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3795                        const src_vi = try isel.use(ty_op.operand);
 3796                        var src_hi64_it = src_vi.field(ty, 8, 8);
 3797                        const src_hi64_vi = try src_hi64_it.only(isel);
 3798                        const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 3799                        var src_lo64_it = src_vi.field(ty, 0, 8);
 3800                        const src_lo64_vi = try src_lo64_it.only(isel);
 3801                        const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
 3802                        const lo64_ra = try isel.allocIntReg();
 3803                        defer isel.freeReg(lo64_ra);
 3804                        const hi64_ra = try isel.allocIntReg();
 3805                        defer isel.freeReg(hi64_ra);
 3806                        try isel.emit(.csel(res_ra.w(), lo64_ra.w(), hi64_ra.w(), .ne));
 3807                        try isel.emit(.add(hi64_ra.w(), hi64_ra.w(), .{ .immediate = 64 }));
 3808                        try isel.emit(.subs(.xzr, src_lo64_mat.ra.x(), .{ .immediate = 0 }));
 3809                        try isel.ctzLimb(hi64_ra, .{ .signedness = .unsigned, .bits = 64 }, src_hi64_mat.ra);
 3810                        try isel.ctzLimb(lo64_ra, .{ .signedness = int_info.signedness, .bits = bits - 64 }, src_lo64_mat.ra);
 3811                        try src_hi64_mat.finish(isel);
 3812                        try src_lo64_mat.finish(isel);
 3813                    },
 3814                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 3815                }
 3816            }
 3817            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3818        },
 3819        .popcount => |air_tag| {
 3820            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3821                defer res_vi.value.deref(isel);
 3822
 3823                const ty_op = air.data(air.inst_index).ty_op;
 3824                const ty = isel.air.typeOf(ty_op.operand, ip);
 3825                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3826                const int_info = ty.intInfo(zcu);
 3827                if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3828
 3829                const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3830                const src_vi = try isel.use(ty_op.operand);
 3831                const src_mat = try src_vi.matReg(isel);
 3832                const vec_ra = try isel.allocVecReg();
 3833                defer isel.freeReg(vec_ra);
 3834                try isel.emit(.umov(res_ra.w(), vec_ra.@"b[]"(0)));
 3835                switch (int_info.bits) {
 3836                    else => unreachable,
 3837                    1...8 => {},
 3838                    9...16 => try isel.emit(.addp(vec_ra.@"8b"(), vec_ra.@"8b"(), .{ .vector = vec_ra.@"8b"() })),
 3839                    17...64 => try isel.emit(.addv(vec_ra.b(), vec_ra.@"8b"())),
 3840                }
 3841                try isel.emit(.cnt(vec_ra.@"8b"(), vec_ra.@"8b"()));
 3842                switch (int_info.bits) {
 3843                    else => unreachable,
 3844                    1...31 => |bits| switch (int_info.signedness) {
 3845                        .signed => {
 3846                            try isel.emit(.fmov(vec_ra.s(), .{ .register = res_ra.w() }));
 3847                            try isel.emit(.ubfm(res_ra.w(), src_mat.ra.w(), .{
 3848                                .N = .word,
 3849                                .immr = 0,
 3850                                .imms = @intCast(bits - 1),
 3851                            }));
 3852                        },
 3853                        .unsigned => try isel.emit(.fmov(vec_ra.s(), .{ .register = src_mat.ra.w() })),
 3854                    },
 3855                    32 => try isel.emit(.fmov(vec_ra.s(), .{ .register = src_mat.ra.w() })),
 3856                    33...63 => |bits| switch (int_info.signedness) {
 3857                        .signed => {
 3858                            try isel.emit(.fmov(vec_ra.d(), .{ .register = res_ra.x() }));
 3859                            try isel.emit(.ubfm(res_ra.x(), src_mat.ra.x(), .{
 3860                                .N = .doubleword,
 3861                                .immr = 0,
 3862                                .imms = @intCast(bits - 1),
 3863                            }));
 3864                        },
 3865                        .unsigned => try isel.emit(.fmov(vec_ra.d(), .{ .register = src_mat.ra.x() })),
 3866                    },
 3867                    64 => try isel.emit(.fmov(vec_ra.d(), .{ .register = src_mat.ra.x() })),
 3868                }
 3869                try src_mat.finish(isel);
 3870            }
 3871            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3872        },
 3873        .byte_swap => |air_tag| {
 3874            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3875                defer res_vi.value.deref(isel);
 3876
 3877                const ty_op = air.data(air.inst_index).ty_op;
 3878                const ty = ty_op.ty.toType();
 3879                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3880                const int_info = ty.intInfo(zcu);
 3881                if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3882
 3883                if (int_info.bits == 8) break :unused try res_vi.value.move(isel, ty_op.operand);
 3884                const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3885                const src_vi = try isel.use(ty_op.operand);
 3886                const src_mat = try src_vi.matReg(isel);
 3887                switch (int_info.bits) {
 3888                    else => unreachable,
 3889                    16 => switch (int_info.signedness) {
 3890                        .signed => {
 3891                            try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
 3892                                .N = .word,
 3893                                .immr = 32 - 16,
 3894                                .imms = 32 - 1,
 3895                            }));
 3896                            try isel.emit(.rev(res_ra.w(), src_mat.ra.w()));
 3897                        },
 3898                        .unsigned => try isel.emit(.rev16(res_ra.w(), src_mat.ra.w())),
 3899                    },
 3900                    24 => {
 3901                        switch (int_info.signedness) {
 3902                            .signed => try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
 3903                                .N = .word,
 3904                                .immr = 32 - 24,
 3905                                .imms = 32 - 1,
 3906                            })),
 3907                            .unsigned => try isel.emit(.ubfm(res_ra.w(), res_ra.w(), .{
 3908                                .N = .word,
 3909                                .immr = 32 - 24,
 3910                                .imms = 32 - 1,
 3911                            })),
 3912                        }
 3913                        try isel.emit(.rev(res_ra.w(), src_mat.ra.w()));
 3914                    },
 3915                    32 => try isel.emit(.rev(res_ra.w(), src_mat.ra.w())),
 3916                    40, 48, 56 => |bits| {
 3917                        switch (int_info.signedness) {
 3918                            .signed => try isel.emit(.sbfm(res_ra.x(), res_ra.x(), .{
 3919                                .N = .doubleword,
 3920                                .immr = @intCast(64 - bits),
 3921                                .imms = 64 - 1,
 3922                            })),
 3923                            .unsigned => try isel.emit(.ubfm(res_ra.x(), res_ra.x(), .{
 3924                                .N = .doubleword,
 3925                                .immr = @intCast(64 - bits),
 3926                                .imms = 64 - 1,
 3927                            })),
 3928                        }
 3929                        try isel.emit(.rev(res_ra.x(), src_mat.ra.x()));
 3930                    },
 3931                    64 => try isel.emit(.rev(res_ra.x(), src_mat.ra.x())),
 3932                }
 3933                try src_mat.finish(isel);
 3934            }
 3935            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3936        },
 3937        .bit_reverse => |air_tag| {
 3938            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3939                defer res_vi.value.deref(isel);
 3940
 3941                const ty_op = air.data(air.inst_index).ty_op;
 3942                const ty = ty_op.ty.toType();
 3943                if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3944                const int_info = ty.intInfo(zcu);
 3945                if (int_info.bits > 64) return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) });
 3946
 3947                const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3948                const src_vi = try isel.use(ty_op.operand);
 3949                const src_mat = try src_vi.matReg(isel);
 3950                switch (int_info.bits) {
 3951                    else => unreachable,
 3952                    1...31 => |bits| {
 3953                        switch (int_info.signedness) {
 3954                            .signed => try isel.emit(.sbfm(res_ra.w(), res_ra.w(), .{
 3955                                .N = .word,
 3956                                .immr = @intCast(32 - bits),
 3957                                .imms = 32 - 1,
 3958                            })),
 3959                            .unsigned => try isel.emit(.ubfm(res_ra.w(), res_ra.w(), .{
 3960                                .N = .word,
 3961                                .immr = @intCast(32 - bits),
 3962                                .imms = 32 - 1,
 3963                            })),
 3964                        }
 3965                        try isel.emit(.rbit(res_ra.w(), src_mat.ra.w()));
 3966                    },
 3967                    32 => try isel.emit(.rbit(res_ra.w(), src_mat.ra.w())),
 3968                    33...63 => |bits| {
 3969                        switch (int_info.signedness) {
 3970                            .signed => try isel.emit(.sbfm(res_ra.x(), res_ra.x(), .{
 3971                                .N = .doubleword,
 3972                                .immr = @intCast(64 - bits),
 3973                                .imms = 64 - 1,
 3974                            })),
 3975                            .unsigned => try isel.emit(.ubfm(res_ra.x(), res_ra.x(), .{
 3976                                .N = .doubleword,
 3977                                .immr = @intCast(64 - bits),
 3978                                .imms = 64 - 1,
 3979                            })),
 3980                        }
 3981                        try isel.emit(.rbit(res_ra.x(), src_mat.ra.x()));
 3982                    },
 3983                    64 => try isel.emit(.rbit(res_ra.x(), src_mat.ra.x())),
 3984                }
 3985                try src_mat.finish(isel);
 3986            }
 3987            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 3988        },
 3989        .sqrt, .floor, .ceil, .round, .trunc_float => |air_tag| {
 3990            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 3991                defer res_vi.value.deref(isel);
 3992
 3993                const un_op = air.data(air.inst_index).un_op;
 3994                const ty = isel.air.typeOf(un_op, ip);
 3995                switch (ty.floatBits(isel.target)) {
 3996                    else => unreachable,
 3997                    16, 32, 64 => |bits| {
 3998                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 3999                        const need_fcvt = switch (bits) {
 4000                            else => unreachable,
 4001                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 4002                            32, 64 => false,
 4003                        };
 4004                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 4005                        const src_vi = try isel.use(un_op);
 4006                        const src_mat = try src_vi.matReg(isel);
 4007                        const src_ra = if (need_fcvt) try isel.allocVecReg() else src_mat.ra;
 4008                        defer if (need_fcvt) isel.freeReg(src_ra);
 4009                        try isel.emit(bits: switch (bits) {
 4010                            else => unreachable,
 4011                            16 => if (need_fcvt) continue :bits 32 else switch (air_tag) {
 4012                                else => unreachable,
 4013                                .sqrt => .fsqrt(res_ra.h(), src_ra.h()),
 4014                                .floor => .frintm(res_ra.h(), src_ra.h()),
 4015                                .ceil => .frintp(res_ra.h(), src_ra.h()),
 4016                                .round => .frinta(res_ra.h(), src_ra.h()),
 4017                                .trunc_float => .frintz(res_ra.h(), src_ra.h()),
 4018                            },
 4019                            32 => switch (air_tag) {
 4020                                else => unreachable,
 4021                                .sqrt => .fsqrt(res_ra.s(), src_ra.s()),
 4022                                .floor => .frintm(res_ra.s(), src_ra.s()),
 4023                                .ceil => .frintp(res_ra.s(), src_ra.s()),
 4024                                .round => .frinta(res_ra.s(), src_ra.s()),
 4025                                .trunc_float => .frintz(res_ra.s(), src_ra.s()),
 4026                            },
 4027                            64 => switch (air_tag) {
 4028                                else => unreachable,
 4029                                .sqrt => .fsqrt(res_ra.d(), src_ra.d()),
 4030                                .floor => .frintm(res_ra.d(), src_ra.d()),
 4031                                .ceil => .frintp(res_ra.d(), src_ra.d()),
 4032                                .round => .frinta(res_ra.d(), src_ra.d()),
 4033                                .trunc_float => .frintz(res_ra.d(), src_ra.d()),
 4034                            },
 4035                        });
 4036                        if (need_fcvt) try isel.emit(.fcvt(src_ra.s(), src_mat.ra.h()));
 4037                        try src_mat.finish(isel);
 4038                    },
 4039                    80, 128 => |bits| {
 4040                        try call.prepareReturn(isel);
 4041                        switch (bits) {
 4042                            else => unreachable,
 4043                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 4044                            80 => {
 4045                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 4046                                const res_hi16_vi = try res_hi16_it.only(isel);
 4047                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 4048                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 4049                                const res_lo64_vi = try res_lo64_it.only(isel);
 4050                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 4051                            },
 4052                        }
 4053                        try call.finishReturn(isel);
 4054
 4055                        try call.prepareCallee(isel);
 4056                        try isel.global_relocs.append(gpa, .{
 4057                            .name = switch (air_tag) {
 4058                                else => unreachable,
 4059                                .sqrt => switch (bits) {
 4060                                    else => unreachable,
 4061                                    16 => "__sqrth",
 4062                                    32 => "sqrtf",
 4063                                    64 => "sqrt",
 4064                                    80 => "__sqrtx",
 4065                                    128 => "sqrtq",
 4066                                },
 4067                                .floor => switch (bits) {
 4068                                    else => unreachable,
 4069                                    16 => "__floorh",
 4070                                    32 => "floorf",
 4071                                    64 => "floor",
 4072                                    80 => "__floorx",
 4073                                    128 => "floorq",
 4074                                },
 4075                                .ceil => switch (bits) {
 4076                                    else => unreachable,
 4077                                    16 => "__ceilh",
 4078                                    32 => "ceilf",
 4079                                    64 => "ceil",
 4080                                    80 => "__ceilx",
 4081                                    128 => "ceilq",
 4082                                },
 4083                                .round => switch (bits) {
 4084                                    else => unreachable,
 4085                                    16 => "__roundh",
 4086                                    32 => "roundf",
 4087                                    64 => "round",
 4088                                    80 => "__roundx",
 4089                                    128 => "roundq",
 4090                                },
 4091                                .trunc_float => switch (bits) {
 4092                                    else => unreachable,
 4093                                    16 => "__trunch",
 4094                                    32 => "truncf",
 4095                                    64 => "trunc",
 4096                                    80 => "__truncx",
 4097                                    128 => "truncq",
 4098                                },
 4099                            },
 4100                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 4101                        });
 4102                        try isel.emit(.bl(0));
 4103                        try call.finishCallee(isel);
 4104
 4105                        try call.prepareParams(isel);
 4106                        const src_vi = try isel.use(un_op);
 4107                        switch (bits) {
 4108                            else => unreachable,
 4109                            16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
 4110                            80 => {
 4111                                var src_hi16_it = src_vi.field(ty, 8, 8);
 4112                                const src_hi16_vi = try src_hi16_it.only(isel);
 4113                                try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
 4114                                var src_lo64_it = src_vi.field(ty, 0, 8);
 4115                                const src_lo64_vi = try src_lo64_it.only(isel);
 4116                                try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
 4117                            },
 4118                        }
 4119                        try call.finishParams(isel);
 4120                    },
 4121                }
 4122            }
 4123            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4124        },
 4125        .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10 => |air_tag| {
 4126            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| {
 4127                defer res_vi.value.deref(isel);
 4128
 4129                const un_op = air.data(air.inst_index).un_op;
 4130                const ty = isel.air.typeOf(un_op, ip);
 4131                const bits = ty.floatBits(isel.target);
 4132                try call.prepareReturn(isel);
 4133                switch (bits) {
 4134                    else => unreachable,
 4135                    16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 4136                    80 => {
 4137                        var res_hi16_it = res_vi.value.field(ty, 8, 8);
 4138                        const res_hi16_vi = try res_hi16_it.only(isel);
 4139                        try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 4140                        var res_lo64_it = res_vi.value.field(ty, 0, 8);
 4141                        const res_lo64_vi = try res_lo64_it.only(isel);
 4142                        try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 4143                    },
 4144                }
 4145                try call.finishReturn(isel);
 4146
 4147                try call.prepareCallee(isel);
 4148                try isel.global_relocs.append(gpa, .{
 4149                    .name = switch (air_tag) {
 4150                        else => unreachable,
 4151                        .sin => switch (bits) {
 4152                            else => unreachable,
 4153                            16 => "__sinh",
 4154                            32 => "sinf",
 4155                            64 => "sin",
 4156                            80 => "__sinx",
 4157                            128 => "sinq",
 4158                        },
 4159                        .cos => switch (bits) {
 4160                            else => unreachable,
 4161                            16 => "__cosh",
 4162                            32 => "cosf",
 4163                            64 => "cos",
 4164                            80 => "__cosx",
 4165                            128 => "cosq",
 4166                        },
 4167                        .tan => switch (bits) {
 4168                            else => unreachable,
 4169                            16 => "__tanh",
 4170                            32 => "tanf",
 4171                            64 => "tan",
 4172                            80 => "__tanx",
 4173                            128 => "tanq",
 4174                        },
 4175                        .exp => switch (bits) {
 4176                            else => unreachable,
 4177                            16 => "__exph",
 4178                            32 => "expf",
 4179                            64 => "exp",
 4180                            80 => "__expx",
 4181                            128 => "expq",
 4182                        },
 4183                        .exp2 => switch (bits) {
 4184                            else => unreachable,
 4185                            16 => "__exp2h",
 4186                            32 => "exp2f",
 4187                            64 => "exp2",
 4188                            80 => "__exp2x",
 4189                            128 => "exp2q",
 4190                        },
 4191                        .log => switch (bits) {
 4192                            else => unreachable,
 4193                            16 => "__logh",
 4194                            32 => "logf",
 4195                            64 => "log",
 4196                            80 => "__logx",
 4197                            128 => "logq",
 4198                        },
 4199                        .log2 => switch (bits) {
 4200                            else => unreachable,
 4201                            16 => "__log2h",
 4202                            32 => "log2f",
 4203                            64 => "log2",
 4204                            80 => "__log2x",
 4205                            128 => "log2q",
 4206                        },
 4207                        .log10 => switch (bits) {
 4208                            else => unreachable,
 4209                            16 => "__log10h",
 4210                            32 => "log10f",
 4211                            64 => "log10",
 4212                            80 => "__log10x",
 4213                            128 => "log10q",
 4214                        },
 4215                    },
 4216                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 4217                });
 4218                try isel.emit(.bl(0));
 4219                try call.finishCallee(isel);
 4220
 4221                try call.prepareParams(isel);
 4222                const src_vi = try isel.use(un_op);
 4223                switch (bits) {
 4224                    else => unreachable,
 4225                    16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
 4226                    80 => {
 4227                        var src_hi16_it = src_vi.field(ty, 8, 8);
 4228                        const src_hi16_vi = try src_hi16_it.only(isel);
 4229                        try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
 4230                        var src_lo64_it = src_vi.field(ty, 0, 8);
 4231                        const src_lo64_vi = try src_lo64_it.only(isel);
 4232                        try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
 4233                    },
 4234                }
 4235                try call.finishParams(isel);
 4236            }
 4237            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4238        },
 4239        .abs => |air_tag| {
 4240            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 4241                defer res_vi.value.deref(isel);
 4242
 4243                const ty_op = air.data(air.inst_index).ty_op;
 4244                const ty = ty_op.ty.toType();
 4245                if (!ty.isRuntimeFloat()) {
 4246                    if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ air_tag, isel.fmtType(ty) });
 4247                    switch (ty.intInfo(zcu).bits) {
 4248                        0 => unreachable,
 4249                        1...32 => {
 4250                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4251                            const src_vi = try isel.use(ty_op.operand);
 4252                            const src_mat = try src_vi.matReg(isel);
 4253                            try isel.emit(.csneg(res_ra.w(), src_mat.ra.w(), src_mat.ra.w(), .pl));
 4254                            try isel.emit(.subs(.wzr, src_mat.ra.w(), .{ .immediate = 0 }));
 4255                            try src_mat.finish(isel);
 4256                        },
 4257                        33...64 => {
 4258                            const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4259                            const src_vi = try isel.use(ty_op.operand);
 4260                            const src_mat = try src_vi.matReg(isel);
 4261                            try isel.emit(.csneg(res_ra.x(), src_mat.ra.x(), src_mat.ra.x(), .pl));
 4262                            try isel.emit(.subs(.xzr, src_mat.ra.x(), .{ .immediate = 0 }));
 4263                            try src_mat.finish(isel);
 4264                        },
 4265                        65...128 => {
 4266                            var res_hi64_it = res_vi.value.field(ty, 8, 8);
 4267                            const res_hi64_vi = try res_hi64_it.only(isel);
 4268                            const res_hi64_ra = try res_hi64_vi.?.defReg(isel);
 4269                            var res_lo64_it = res_vi.value.field(ty, 0, 8);
 4270                            const res_lo64_vi = try res_lo64_it.only(isel);
 4271                            const res_lo64_ra = try res_lo64_vi.?.defReg(isel);
 4272                            if (res_hi64_ra == null and res_lo64_ra == null) break :unused;
 4273                            const src_ty = isel.air.typeOf(ty_op.operand, ip);
 4274                            const src_vi = try isel.use(ty_op.operand);
 4275                            var src_hi64_it = src_vi.field(src_ty, 8, 8);
 4276                            const src_hi64_vi = try src_hi64_it.only(isel);
 4277                            const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 4278                            var src_lo64_it = src_vi.field(src_ty, 0, 8);
 4279                            const src_lo64_vi = try src_lo64_it.only(isel);
 4280                            const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
 4281                            const lo64_ra = try isel.allocIntReg();
 4282                            defer isel.freeReg(lo64_ra);
 4283                            const hi64_ra, const mask_ra = alloc_ras: {
 4284                                const res_lo64_lock: RegLock = if (res_lo64_ra) |res_ra| isel.tryLockReg(res_ra) else .empty;
 4285                                defer res_lo64_lock.unlock(isel);
 4286                                break :alloc_ras .{ try isel.allocIntReg(), try isel.allocIntReg() };
 4287                            };
 4288                            defer {
 4289                                isel.freeReg(hi64_ra);
 4290                                isel.freeReg(mask_ra);
 4291                            }
 4292                            if (res_hi64_ra) |res_ra| try isel.emit(.sbc(res_ra.x(), hi64_ra.x(), mask_ra.x()));
 4293                            try isel.emit(.subs(
 4294                                if (res_lo64_ra) |res_ra| res_ra.x() else .xzr,
 4295                                lo64_ra.x(),
 4296                                .{ .register = mask_ra.x() },
 4297                            ));
 4298                            if (res_hi64_ra) |_| try isel.emit(.eor(hi64_ra.x(), src_hi64_mat.ra.x(), .{ .register = mask_ra.x() }));
 4299                            try isel.emit(.eor(lo64_ra.x(), src_lo64_mat.ra.x(), .{ .register = mask_ra.x() }));
 4300                            try isel.emit(.sbfm(mask_ra.x(), src_hi64_mat.ra.x(), .{
 4301                                .N = .doubleword,
 4302                                .immr = 64 - 1,
 4303                                .imms = 64 - 1,
 4304                            }));
 4305                            try src_lo64_mat.finish(isel);
 4306                            try src_hi64_mat.finish(isel);
 4307                        },
 4308                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(ty) }),
 4309                    }
 4310                } else switch (ty.floatBits(isel.target)) {
 4311                    else => unreachable,
 4312                    16 => {
 4313                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4314                        const src_vi = try isel.use(ty_op.operand);
 4315                        const src_mat = try src_vi.matReg(isel);
 4316                        try isel.emit(if (isel.target.cpu.has(.aarch64, .fullfp16))
 4317                            .fabs(res_ra.h(), src_mat.ra.h())
 4318                        else
 4319                            .bic(res_ra.@"4h"(), res_ra.@"4h"(), .{ .shifted_immediate = .{
 4320                                .immediate = 0b10000000,
 4321                                .lsl = 8,
 4322                            } }));
 4323                        try src_mat.finish(isel);
 4324                    },
 4325                    32 => {
 4326                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4327                        const src_vi = try isel.use(ty_op.operand);
 4328                        const src_mat = try src_vi.matReg(isel);
 4329                        try isel.emit(.fabs(res_ra.s(), src_mat.ra.s()));
 4330                        try src_mat.finish(isel);
 4331                    },
 4332                    64 => {
 4333                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4334                        const src_vi = try isel.use(ty_op.operand);
 4335                        const src_mat = try src_vi.matReg(isel);
 4336                        try isel.emit(.fabs(res_ra.d(), src_mat.ra.d()));
 4337                        try src_mat.finish(isel);
 4338                    },
 4339                    80 => {
 4340                        const src_vi = try isel.use(ty_op.operand);
 4341                        var res_hi16_it = res_vi.value.field(ty, 8, 8);
 4342                        const res_hi16_vi = try res_hi16_it.only(isel);
 4343                        if (try res_hi16_vi.?.defReg(isel)) |res_hi16_ra| {
 4344                            var src_hi16_it = src_vi.field(ty, 8, 8);
 4345                            const src_hi16_vi = try src_hi16_it.only(isel);
 4346                            const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
 4347                            try isel.emit(.@"and"(res_hi16_ra.w(), src_hi16_mat.ra.w(), .{ .immediate = .{
 4348                                .N = .word,
 4349                                .immr = 0,
 4350                                .imms = 15 - 1,
 4351                            } }));
 4352                            try src_hi16_mat.finish(isel);
 4353                        }
 4354                        var res_lo64_it = res_vi.value.field(ty, 0, 8);
 4355                        const res_lo64_vi = try res_lo64_it.only(isel);
 4356                        if (try res_lo64_vi.?.defReg(isel)) |res_lo64_ra| {
 4357                            var src_lo64_it = src_vi.field(ty, 0, 8);
 4358                            const src_lo64_vi = try src_lo64_it.only(isel);
 4359                            try src_lo64_vi.?.liveOut(isel, res_lo64_ra);
 4360                        }
 4361                    },
 4362                    128 => {
 4363                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4364                        const src_vi = try isel.use(ty_op.operand);
 4365                        const src_mat = try src_vi.matReg(isel);
 4366                        const neg_zero_ra = try isel.allocVecReg();
 4367                        defer isel.freeReg(neg_zero_ra);
 4368                        try isel.emit(.bic(res_ra.@"16b"(), src_mat.ra.@"16b"(), .{ .register = neg_zero_ra.@"16b"() }));
 4369                        try isel.literals.appendNTimes(gpa, 0, -%isel.literals.items.len % 4);
 4370                        try isel.literal_relocs.append(gpa, .{
 4371                            .label = @intCast(isel.instructions.items.len),
 4372                        });
 4373                        try isel.emit(.ldr(neg_zero_ra.q(), .{
 4374                            .literal = @intCast((isel.instructions.items.len + 1 + isel.literals.items.len) << 2),
 4375                        }));
 4376                        try isel.emitLiteral(&(.{0} ** 15 ++ .{0x80}));
 4377                        try src_mat.finish(isel);
 4378                    },
 4379                }
 4380            }
 4381            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4382        },
 4383        .neg, .neg_optimized => {
 4384            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 4385                defer res_vi.value.deref(isel);
 4386
 4387                const un_op = air.data(air.inst_index).un_op;
 4388                const ty = isel.air.typeOf(un_op, ip);
 4389                switch (ty.floatBits(isel.target)) {
 4390                    else => unreachable,
 4391                    16 => {
 4392                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4393                        const src_vi = try isel.use(un_op);
 4394                        const src_mat = try src_vi.matReg(isel);
 4395                        if (isel.target.cpu.has(.aarch64, .fullfp16)) {
 4396                            try isel.emit(.fneg(res_ra.h(), src_mat.ra.h()));
 4397                        } else {
 4398                            const neg_zero_ra = try isel.allocVecReg();
 4399                            defer isel.freeReg(neg_zero_ra);
 4400                            try isel.emit(.eor(res_ra.@"8b"(), res_ra.@"8b"(), .{ .register = neg_zero_ra.@"8b"() }));
 4401                            try isel.emit(.movi(neg_zero_ra.@"4h"(), 0b10000000, .{ .lsl = 8 }));
 4402                        }
 4403                        try src_mat.finish(isel);
 4404                    },
 4405                    32 => {
 4406                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4407                        const src_vi = try isel.use(un_op);
 4408                        const src_mat = try src_vi.matReg(isel);
 4409                        try isel.emit(.fneg(res_ra.s(), src_mat.ra.s()));
 4410                        try src_mat.finish(isel);
 4411                    },
 4412                    64 => {
 4413                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4414                        const src_vi = try isel.use(un_op);
 4415                        const src_mat = try src_vi.matReg(isel);
 4416                        try isel.emit(.fneg(res_ra.d(), src_mat.ra.d()));
 4417                        try src_mat.finish(isel);
 4418                    },
 4419                    80 => {
 4420                        const src_vi = try isel.use(un_op);
 4421                        var res_hi16_it = res_vi.value.field(ty, 8, 8);
 4422                        const res_hi16_vi = try res_hi16_it.only(isel);
 4423                        if (try res_hi16_vi.?.defReg(isel)) |res_hi16_ra| {
 4424                            var src_hi16_it = src_vi.field(ty, 8, 8);
 4425                            const src_hi16_vi = try src_hi16_it.only(isel);
 4426                            const src_hi16_mat = try src_hi16_vi.?.matReg(isel);
 4427                            try isel.emit(.eor(res_hi16_ra.w(), src_hi16_mat.ra.w(), .{ .immediate = .{
 4428                                .N = .word,
 4429                                .immr = 32 - 15,
 4430                                .imms = 1 - 1,
 4431                            } }));
 4432                            try src_hi16_mat.finish(isel);
 4433                        }
 4434                        var res_lo64_it = res_vi.value.field(ty, 0, 8);
 4435                        const res_lo64_vi = try res_lo64_it.only(isel);
 4436                        if (try res_lo64_vi.?.defReg(isel)) |res_lo64_ra| {
 4437                            var src_lo64_it = src_vi.field(ty, 0, 8);
 4438                            const src_lo64_vi = try src_lo64_it.only(isel);
 4439                            try src_lo64_vi.?.liveOut(isel, res_lo64_ra);
 4440                        }
 4441                    },
 4442                    128 => {
 4443                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 4444                        const src_vi = try isel.use(un_op);
 4445                        const src_mat = try src_vi.matReg(isel);
 4446                        const neg_zero_ra = try isel.allocVecReg();
 4447                        defer isel.freeReg(neg_zero_ra);
 4448                        try isel.emit(.eor(res_ra.@"16b"(), src_mat.ra.@"16b"(), .{ .register = neg_zero_ra.@"16b"() }));
 4449                        try isel.literals.appendNTimes(gpa, 0, -%isel.literals.items.len % 4);
 4450                        try isel.literal_relocs.append(gpa, .{
 4451                            .label = @intCast(isel.instructions.items.len),
 4452                        });
 4453                        try isel.emit(.ldr(neg_zero_ra.q(), .{
 4454                            .literal = @intCast((isel.instructions.items.len + 1 + isel.literals.items.len) << 2),
 4455                        }));
 4456                        try isel.emitLiteral(&(.{0} ** 15 ++ .{0x80}));
 4457                        try src_mat.finish(isel);
 4458                    },
 4459                }
 4460            }
 4461            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4462        },
 4463        .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => |air_tag| {
 4464            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 4465                defer res_vi.value.deref(isel);
 4466
 4467                const bin_op = air.data(air.inst_index).bin_op;
 4468                const ty = isel.air.typeOf(bin_op.lhs, ip);
 4469                switch (ip.indexToKey(ty.toIntern())) {
 4470                    else => {},
 4471                    .opt_type => |payload_ty| switch (air_tag) {
 4472                        else => unreachable,
 4473                        .cmp_eq, .cmp_neq => if (!ty.optionalReprIsPayload(zcu)) {
 4474                            const lhs_vi = try isel.use(bin_op.lhs);
 4475                            const rhs_vi = try isel.use(bin_op.rhs);
 4476                            const payload_size = ZigType.abiSize(.fromInterned(payload_ty), zcu);
 4477                            var lhs_payload_part_it = lhs_vi.field(ty, 0, payload_size);
 4478                            const lhs_payload_part_vi = try lhs_payload_part_it.only(isel);
 4479                            var rhs_payload_part_it = rhs_vi.field(ty, 0, payload_size);
 4480                            const rhs_payload_part_vi = try rhs_payload_part_it.only(isel);
 4481                            const cmp_info = try isel.cmp(
 4482                                try res_vi.value.defReg(isel) orelse break :unused,
 4483                                .fromInterned(payload_ty),
 4484                                lhs_payload_part_vi.?,
 4485                                air_tag.toCmpOp().?,
 4486                                rhs_payload_part_vi.?,
 4487                            );
 4488                            try isel.emit(.@"b."(
 4489                                .vc,
 4490                                @intCast((isel.instructions.items.len + 1 - cmp_info.cset_label) << 2),
 4491                            ));
 4492                            var lhs_has_value_part_it = lhs_vi.field(ty, payload_size, 1);
 4493                            const lhs_has_value_part_vi = try lhs_has_value_part_it.only(isel);
 4494                            const lhs_has_value_part_mat = try lhs_has_value_part_vi.?.matReg(isel);
 4495                            var rhs_has_value_part_it = rhs_vi.field(ty, payload_size, 1);
 4496                            const rhs_has_value_part_vi = try rhs_has_value_part_it.only(isel);
 4497                            const rhs_has_value_part_mat = try rhs_has_value_part_vi.?.matReg(isel);
 4498                            try isel.emit(.ccmp(
 4499                                lhs_has_value_part_mat.ra.w(),
 4500                                .{ .register = rhs_has_value_part_mat.ra.w() },
 4501                                .{ .n = false, .z = false, .c = false, .v = true },
 4502                                .eq,
 4503                            ));
 4504                            try isel.emit(.ands(
 4505                                .wzr,
 4506                                lhs_has_value_part_mat.ra.w(),
 4507                                .{ .register = rhs_has_value_part_mat.ra.w() },
 4508                            ));
 4509                            try rhs_has_value_part_mat.finish(isel);
 4510                            try lhs_has_value_part_mat.finish(isel);
 4511                            break :unused;
 4512                        },
 4513                    },
 4514                }
 4515                _ = try isel.cmp(
 4516                    try res_vi.value.defReg(isel) orelse break :unused,
 4517                    ty,
 4518                    try isel.use(bin_op.lhs),
 4519                    air_tag.toCmpOp().?,
 4520                    try isel.use(bin_op.rhs),
 4521                );
 4522            }
 4523            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4524        },
 4525        .cond_br => {
 4526            const pl_op = air.data(air.inst_index).pl_op;
 4527            const extra = isel.air.extraData(Air.CondBr, pl_op.payload);
 4528
 4529            try isel.body(@ptrCast(isel.air.extra.items[extra.end + extra.data.then_body_len ..][0..extra.data.else_body_len]));
 4530            const else_label = isel.instructions.items.len;
 4531            const else_live_registers = isel.live_registers;
 4532            try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.then_body_len]));
 4533            try isel.merge(&else_live_registers, .{});
 4534
 4535            const cond_vi = try isel.use(pl_op.operand);
 4536            const cond_mat = try cond_vi.matReg(isel);
 4537            try isel.emit(.tbz(
 4538                cond_mat.ra.x(),
 4539                0,
 4540                @intCast((isel.instructions.items.len + 1 - else_label) << 2),
 4541            ));
 4542            try cond_mat.finish(isel);
 4543
 4544            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4545        },
 4546        .switch_br => {
 4547            const switch_br = isel.air.unwrapSwitch(air.inst_index);
 4548            const cond_ty = isel.air.typeOf(switch_br.operand, ip);
 4549            const cond_int_info: std.builtin.Type.Int = if (cond_ty.toIntern() == .bool_type)
 4550                .{ .signedness = .unsigned, .bits = 1 }
 4551            else if (cond_ty.isAbiInt(zcu))
 4552                cond_ty.intInfo(zcu)
 4553            else
 4554                return isel.fail("bad switch cond {f}", .{isel.fmtType(cond_ty)});
 4555
 4556            var final_case = true;
 4557            if (switch_br.else_body_len > 0) {
 4558                var cases_it = switch_br.iterateCases();
 4559                while (cases_it.next()) |_| {}
 4560                try isel.body(cases_it.elseBody());
 4561                assert(final_case);
 4562                final_case = false;
 4563            }
 4564            const zero_reg: Register = switch (cond_int_info.bits) {
 4565                else => unreachable,
 4566                1...32 => .wzr,
 4567                33...64 => .xzr,
 4568            };
 4569            var cond_mat: ?Value.Materialize = null;
 4570            var cond_reg: Register = undefined;
 4571            var cases_it = switch_br.iterateCases();
 4572            while (cases_it.next()) |case| {
 4573                const next_label = isel.instructions.items.len;
 4574                const next_live_registers = isel.live_registers;
 4575                try isel.body(case.body);
 4576                if (final_case) {
 4577                    final_case = false;
 4578                    continue;
 4579                }
 4580                try isel.merge(&next_live_registers, .{});
 4581                if (cond_mat == null) {
 4582                    var cond_vi = try isel.use(switch_br.operand);
 4583                    cond_mat = try cond_vi.matReg(isel);
 4584                    cond_reg = switch (cond_int_info.bits) {
 4585                        else => unreachable,
 4586                        1...32 => cond_mat.?.ra.w(),
 4587                        33...64 => cond_mat.?.ra.x(),
 4588                    };
 4589                }
 4590                if (case.ranges.len == 0 and case.items.len == 1 and Constant.fromInterned(
 4591                    case.items[0].toInterned().?,
 4592                ).orderAgainstZero(zcu).compare(.eq)) {
 4593                    try isel.emit(.cbnz(
 4594                        cond_reg,
 4595                        @intCast((isel.instructions.items.len + 1 - next_label) << 2),
 4596                    ));
 4597                    continue;
 4598                }
 4599                try isel.emit(.@"b."(
 4600                    .invert(switch (case.ranges.len) {
 4601                        0 => .eq,
 4602                        else => .ls,
 4603                    }),
 4604                    @intCast((isel.instructions.items.len + 1 - next_label) << 2),
 4605                ));
 4606                var case_range_index = case.ranges.len;
 4607                while (case_range_index > 0) {
 4608                    case_range_index -= 1;
 4609
 4610                    const low_val: Constant = .fromInterned(case.ranges[case_range_index][0].toInterned().?);
 4611                    var low_bigint_space: Constant.BigIntSpace = undefined;
 4612                    const low_bigint = low_val.toBigInt(&low_bigint_space, zcu);
 4613                    const low_int: i64 = if (low_bigint.positive) @bitCast(
 4614                        low_bigint.toInt(u64) catch
 4615                            return isel.fail("too big case range start: {f}", .{isel.fmtConstant(low_val)}),
 4616                    ) else low_bigint.toInt(i64) catch
 4617                        return isel.fail("too big case range start: {f}", .{isel.fmtConstant(low_val)});
 4618
 4619                    const high_val: Constant = .fromInterned(case.ranges[case_range_index][1].toInterned().?);
 4620                    var high_bigint_space: Constant.BigIntSpace = undefined;
 4621                    const high_bigint = high_val.toBigInt(&high_bigint_space, zcu);
 4622                    const high_int: i64 = if (high_bigint.positive) @bitCast(
 4623                        high_bigint.toInt(u64) catch
 4624                            return isel.fail("too big case range end: {f}", .{isel.fmtConstant(high_val)}),
 4625                    ) else high_bigint.toInt(i64) catch
 4626                        return isel.fail("too big case range end: {f}", .{isel.fmtConstant(high_val)});
 4627
 4628                    const adjusted_ra = switch (low_int) {
 4629                        0 => cond_mat.?.ra,
 4630                        else => try isel.allocIntReg(),
 4631                    };
 4632                    defer if (adjusted_ra != cond_mat.?.ra) isel.freeReg(adjusted_ra);
 4633                    const adjusted_reg = switch (cond_int_info.bits) {
 4634                        else => unreachable,
 4635                        1...32 => adjusted_ra.w(),
 4636                        33...64 => adjusted_ra.x(),
 4637                    };
 4638                    const delta_int = high_int -% low_int;
 4639                    if (case_range_index | case.items.len > 0) {
 4640                        if (std.math.cast(u5, delta_int)) |pos_imm| try isel.emit(.ccmp(
 4641                            adjusted_reg,
 4642                            .{ .immediate = pos_imm },
 4643                            .{ .n = false, .z = true, .c = false, .v = false },
 4644                            if (case_range_index > 0) .hi else .ne,
 4645                        )) else if (std.math.cast(u5, -delta_int)) |neg_imm| try isel.emit(.ccmn(
 4646                            adjusted_reg,
 4647                            .{ .immediate = neg_imm },
 4648                            .{ .n = false, .z = true, .c = false, .v = false },
 4649                            if (case_range_index > 0) .hi else .ne,
 4650                        )) else {
 4651                            const imm_ra = try isel.allocIntReg();
 4652                            defer isel.freeReg(imm_ra);
 4653                            const imm_reg = switch (cond_int_info.bits) {
 4654                                else => unreachable,
 4655                                1...32 => imm_ra.w(),
 4656                                33...64 => imm_ra.x(),
 4657                            };
 4658                            try isel.emit(.ccmp(
 4659                                cond_reg,
 4660                                .{ .register = imm_reg },
 4661                                .{ .n = false, .z = true, .c = false, .v = false },
 4662                                if (case_range_index > 0) .hi else .ne,
 4663                            ));
 4664                            try isel.movImmediate(imm_reg, @bitCast(delta_int));
 4665                        }
 4666                    } else {
 4667                        if (std.math.cast(u12, delta_int)) |pos_imm| try isel.emit(.subs(
 4668                            zero_reg,
 4669                            adjusted_reg,
 4670                            .{ .immediate = pos_imm },
 4671                        )) else if (std.math.cast(u12, -delta_int)) |neg_imm| try isel.emit(.adds(
 4672                            zero_reg,
 4673                            adjusted_reg,
 4674                            .{ .immediate = neg_imm },
 4675                        )) else if (if (@as(i12, @truncate(delta_int)) == 0)
 4676                            std.math.cast(u12, delta_int >> 12)
 4677                        else
 4678                            null) |pos_imm_lsr_12| try isel.emit(.subs(
 4679                            zero_reg,
 4680                            adjusted_reg,
 4681                            .{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
 4682                        )) else if (if (@as(i12, @truncate(-delta_int)) == 0)
 4683                            std.math.cast(u12, -delta_int >> 12)
 4684                        else
 4685                            null) |neg_imm_lsr_12| try isel.emit(.adds(
 4686                            zero_reg,
 4687                            adjusted_reg,
 4688                            .{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
 4689                        )) else {
 4690                            const imm_ra = try isel.allocIntReg();
 4691                            defer isel.freeReg(imm_ra);
 4692                            const imm_reg = switch (cond_int_info.bits) {
 4693                                else => unreachable,
 4694                                1...32 => imm_ra.w(),
 4695                                33...64 => imm_ra.x(),
 4696                            };
 4697                            try isel.emit(.subs(zero_reg, adjusted_reg, .{ .register = imm_reg }));
 4698                            try isel.movImmediate(imm_reg, @bitCast(delta_int));
 4699                        }
 4700                    }
 4701
 4702                    switch (low_int) {
 4703                        0 => {},
 4704                        else => {
 4705                            if (std.math.cast(u12, low_int)) |pos_imm| try isel.emit(.sub(
 4706                                adjusted_reg,
 4707                                cond_reg,
 4708                                .{ .immediate = pos_imm },
 4709                            )) else if (std.math.cast(u12, -low_int)) |neg_imm| try isel.emit(.add(
 4710                                adjusted_reg,
 4711                                cond_reg,
 4712                                .{ .immediate = neg_imm },
 4713                            )) else if (if (@as(i12, @truncate(low_int)) == 0)
 4714                                std.math.cast(u12, low_int >> 12)
 4715                            else
 4716                                null) |pos_imm_lsr_12| try isel.emit(.sub(
 4717                                adjusted_reg,
 4718                                cond_reg,
 4719                                .{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
 4720                            )) else if (if (@as(i12, @truncate(-low_int)) == 0)
 4721                                std.math.cast(u12, -low_int >> 12)
 4722                            else
 4723                                null) |neg_imm_lsr_12| try isel.emit(.add(
 4724                                adjusted_reg,
 4725                                cond_reg,
 4726                                .{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
 4727                            )) else {
 4728                                const imm_ra = try isel.allocIntReg();
 4729                                defer isel.freeReg(imm_ra);
 4730                                const imm_reg = switch (cond_int_info.bits) {
 4731                                    else => unreachable,
 4732                                    1...32 => imm_ra.w(),
 4733                                    33...64 => imm_ra.x(),
 4734                                };
 4735                                try isel.emit(.sub(adjusted_reg, cond_reg, .{ .register = imm_reg }));
 4736                                try isel.movImmediate(imm_reg, @bitCast(low_int));
 4737                            }
 4738                        },
 4739                    }
 4740                }
 4741                var case_item_index = case.items.len;
 4742                while (case_item_index > 0) {
 4743                    case_item_index -= 1;
 4744
 4745                    const item_val: Constant = .fromInterned(case.items[case_item_index].toInterned().?);
 4746                    var item_bigint_space: Constant.BigIntSpace = undefined;
 4747                    const item_bigint = item_val.toBigInt(&item_bigint_space, zcu);
 4748                    const item_int: i64 = if (item_bigint.positive) @bitCast(
 4749                        item_bigint.toInt(u64) catch
 4750                            return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)}),
 4751                    ) else item_bigint.toInt(i64) catch
 4752                        return isel.fail("too big case item: {f}", .{isel.fmtConstant(item_val)});
 4753
 4754                    if (case_item_index > 0) {
 4755                        if (std.math.cast(u5, item_int)) |pos_imm| try isel.emit(.ccmp(
 4756                            cond_reg,
 4757                            .{ .immediate = pos_imm },
 4758                            .{ .n = false, .z = true, .c = false, .v = false },
 4759                            .ne,
 4760                        )) else if (std.math.cast(u5, -item_int)) |neg_imm| try isel.emit(.ccmn(
 4761                            cond_reg,
 4762                            .{ .immediate = neg_imm },
 4763                            .{ .n = false, .z = true, .c = false, .v = false },
 4764                            .ne,
 4765                        )) else {
 4766                            const imm_ra = try isel.allocIntReg();
 4767                            defer isel.freeReg(imm_ra);
 4768                            const imm_reg = switch (cond_int_info.bits) {
 4769                                else => unreachable,
 4770                                1...32 => imm_ra.w(),
 4771                                33...64 => imm_ra.x(),
 4772                            };
 4773                            try isel.emit(.ccmp(
 4774                                cond_reg,
 4775                                .{ .register = imm_reg },
 4776                                .{ .n = false, .z = true, .c = false, .v = false },
 4777                                .ne,
 4778                            ));
 4779                            try isel.movImmediate(imm_reg, @bitCast(item_int));
 4780                        }
 4781                    } else {
 4782                        if (std.math.cast(u12, item_int)) |pos_imm| try isel.emit(.subs(
 4783                            zero_reg,
 4784                            cond_reg,
 4785                            .{ .immediate = pos_imm },
 4786                        )) else if (std.math.cast(u12, -item_int)) |neg_imm| try isel.emit(.adds(
 4787                            zero_reg,
 4788                            cond_reg,
 4789                            .{ .immediate = neg_imm },
 4790                        )) else if (if (@as(i12, @truncate(item_int)) == 0)
 4791                            std.math.cast(u12, item_int >> 12)
 4792                        else
 4793                            null) |pos_imm_lsr_12| try isel.emit(.subs(
 4794                            zero_reg,
 4795                            cond_reg,
 4796                            .{ .shifted_immediate = .{ .immediate = pos_imm_lsr_12, .lsl = .@"12" } },
 4797                        )) else if (if (@as(i12, @truncate(-item_int)) == 0)
 4798                            std.math.cast(u12, -item_int >> 12)
 4799                        else
 4800                            null) |neg_imm_lsr_12| try isel.emit(.adds(
 4801                            zero_reg,
 4802                            cond_reg,
 4803                            .{ .shifted_immediate = .{ .immediate = neg_imm_lsr_12, .lsl = .@"12" } },
 4804                        )) else {
 4805                            const imm_ra = try isel.allocIntReg();
 4806                            defer isel.freeReg(imm_ra);
 4807                            const imm_reg = switch (cond_int_info.bits) {
 4808                                else => unreachable,
 4809                                1...32 => imm_ra.w(),
 4810                                33...64 => imm_ra.x(),
 4811                            };
 4812                            try isel.emit(.subs(zero_reg, cond_reg, .{ .register = imm_reg }));
 4813                            try isel.movImmediate(imm_reg, @bitCast(item_int));
 4814                        }
 4815                    }
 4816                }
 4817            }
 4818            if (cond_mat) |mat| try mat.finish(isel);
 4819            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4820        },
 4821        .@"try", .try_cold => {
 4822            const pl_op = air.data(air.inst_index).pl_op;
 4823            const extra = isel.air.extraData(Air.Try, pl_op.payload);
 4824            const error_union_ty = isel.air.typeOf(pl_op.operand, ip);
 4825            const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
 4826            const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
 4827
 4828            const error_union_vi = try isel.use(pl_op.operand);
 4829            if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
 4830                defer payload_vi.value.deref(isel);
 4831
 4832                var payload_part_it = error_union_vi.field(
 4833                    error_union_ty,
 4834                    codegen.errUnionPayloadOffset(payload_ty, zcu),
 4835                    payload_vi.value.size(isel),
 4836                );
 4837                const payload_part_vi = try payload_part_it.only(isel);
 4838                try payload_vi.value.copy(isel, payload_ty, payload_part_vi.?);
 4839            }
 4840
 4841            const cont_label = isel.instructions.items.len;
 4842            const cont_live_registers = isel.live_registers;
 4843            try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
 4844            try isel.merge(&cont_live_registers, .{});
 4845
 4846            var error_set_part_it = error_union_vi.field(
 4847                error_union_ty,
 4848                codegen.errUnionErrorOffset(payload_ty, zcu),
 4849                ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
 4850            );
 4851            const error_set_part_vi = try error_set_part_it.only(isel);
 4852            const error_set_part_mat = try error_set_part_vi.?.matReg(isel);
 4853            try isel.emit(.cbz(
 4854                error_set_part_mat.ra.w(),
 4855                @intCast((isel.instructions.items.len + 1 - cont_label) << 2),
 4856            ));
 4857            try error_set_part_mat.finish(isel);
 4858
 4859            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4860        },
 4861        .try_ptr, .try_ptr_cold => {
 4862            const ty_pl = air.data(air.inst_index).ty_pl;
 4863            const extra = isel.air.extraData(Air.TryPtr, ty_pl.payload);
 4864            const error_union_ty = isel.air.typeOf(extra.data.ptr, ip).childType(zcu);
 4865            const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
 4866            const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
 4867
 4868            const error_union_ptr_vi = try isel.use(extra.data.ptr);
 4869            const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
 4870            if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
 4871                defer payload_ptr_vi.value.deref(isel);
 4872                switch (codegen.errUnionPayloadOffset(ty_pl.ty.toType().childType(zcu), zcu)) {
 4873                    0 => try payload_ptr_vi.value.move(isel, extra.data.ptr),
 4874                    else => |payload_offset| {
 4875                        const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
 4876                        const lo12: u12 = @truncate(payload_offset >> 0);
 4877                        const hi12: u12 = @intCast(payload_offset >> 12);
 4878                        if (hi12 > 0) try isel.emit(.add(
 4879                            payload_ptr_ra.x(),
 4880                            if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
 4881                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 4882                        ));
 4883                        if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
 4884                    },
 4885                }
 4886            }
 4887
 4888            const cont_label = isel.instructions.items.len;
 4889            const cont_live_registers = isel.live_registers;
 4890            try isel.body(@ptrCast(isel.air.extra.items[extra.end..][0..extra.data.body_len]));
 4891            try isel.merge(&cont_live_registers, .{});
 4892
 4893            const error_set_ra = try isel.allocIntReg();
 4894            defer isel.freeReg(error_set_ra);
 4895            try isel.loadReg(
 4896                error_set_ra,
 4897                ZigType.fromInterned(error_union_info.error_set_type).abiSize(zcu),
 4898                .unsigned,
 4899                error_union_ptr_mat.ra,
 4900                codegen.errUnionErrorOffset(payload_ty, zcu),
 4901            );
 4902            try error_union_ptr_mat.finish(isel);
 4903            try isel.emit(.cbz(
 4904                error_set_ra.w(),
 4905                @intCast((isel.instructions.items.len + 1 - cont_label) << 2),
 4906            ));
 4907
 4908            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4909        },
 4910        .dbg_stmt => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
 4911        .dbg_empty_stmt => {
 4912            try isel.emit(.nop());
 4913            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4914        },
 4915        .dbg_inline_block => {
 4916            const ty_pl = air.data(air.inst_index).ty_pl;
 4917            const extra = isel.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
 4918            try isel.block(air.inst_index, ty_pl.ty.toType(), @ptrCast(
 4919                isel.air.extra.items[extra.end..][0..extra.data.body_len],
 4920            ));
 4921            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4922        },
 4923        .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => {
 4924            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4925        },
 4926        .is_null, .is_non_null => |air_tag| {
 4927            if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
 4928                defer is_vi.value.deref(isel);
 4929                const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
 4930
 4931                const un_op = air.data(air.inst_index).un_op;
 4932                const opt_ty = isel.air.typeOf(un_op, ip);
 4933                const payload_ty = opt_ty.optionalChild(zcu);
 4934                const payload_size = payload_ty.abiSize(zcu);
 4935                const has_value_offset, const has_value_size = if (!opt_ty.optionalReprIsPayload(zcu))
 4936                    .{ payload_size, 1 }
 4937                else if (payload_ty.isSlice(zcu))
 4938                    .{ 0, 8 }
 4939                else
 4940                    .{ 0, payload_size };
 4941
 4942                try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(switch (air_tag) {
 4943                    else => unreachable,
 4944                    .is_null => .eq,
 4945                    .is_non_null => .ne,
 4946                })));
 4947                const opt_vi = try isel.use(un_op);
 4948                var has_value_part_it = opt_vi.field(opt_ty, has_value_offset, has_value_size);
 4949                const has_value_part_vi = try has_value_part_it.only(isel);
 4950                const has_value_part_mat = try has_value_part_vi.?.matReg(isel);
 4951                try isel.emit(switch (has_value_size) {
 4952                    else => unreachable,
 4953                    1...4 => .subs(.wzr, has_value_part_mat.ra.w(), .{ .immediate = 0 }),
 4954                    5...8 => .subs(.xzr, has_value_part_mat.ra.x(), .{ .immediate = 0 }),
 4955                });
 4956                try has_value_part_mat.finish(isel);
 4957            }
 4958            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4959        },
 4960        .is_err, .is_non_err => |air_tag| {
 4961            if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
 4962                defer is_vi.value.deref(isel);
 4963                const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
 4964
 4965                const un_op = air.data(air.inst_index).un_op;
 4966                const error_union_ty = isel.air.typeOf(un_op, ip);
 4967                const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
 4968                const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
 4969                const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
 4970                const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
 4971                const error_set_size = error_set_ty.abiSize(zcu);
 4972
 4973                try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(switch (air_tag) {
 4974                    else => unreachable,
 4975                    .is_err => .ne,
 4976                    .is_non_err => .eq,
 4977                })));
 4978                const error_union_vi = try isel.use(un_op);
 4979                var error_set_part_it = error_union_vi.field(error_union_ty, error_set_offset, error_set_size);
 4980                const error_set_part_vi = try error_set_part_it.only(isel);
 4981                const error_set_part_mat = try error_set_part_vi.?.matReg(isel);
 4982                try isel.emit(.ands(.wzr, error_set_part_mat.ra.w(), .{ .immediate = .{
 4983                    .N = .word,
 4984                    .immr = 0,
 4985                    .imms = @intCast(8 * error_set_size - 1),
 4986                } }));
 4987                try error_set_part_mat.finish(isel);
 4988            }
 4989            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 4990        },
 4991        .load => {
 4992            const ty_op = air.data(air.inst_index).ty_op;
 4993            const ptr_ty = isel.air.typeOf(ty_op.operand, ip);
 4994            const ptr_info = ptr_ty.ptrInfo(zcu);
 4995            if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
 4996
 4997            if (ptr_info.flags.is_volatile) _ = try isel.use(air.inst_index.toRef());
 4998            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 4999                defer dst_vi.value.deref(isel);
 5000                const size = dst_vi.value.size(isel);
 5001                if (size <= Value.max_parts and ip.zigTypeTag(ptr_info.child) != .@"union") {
 5002                    const ptr_vi = try isel.use(ty_op.operand);
 5003                    const ptr_mat = try ptr_vi.matReg(isel);
 5004                    _ = try dst_vi.value.load(isel, ty_op.ty.toType(), ptr_mat.ra, .{
 5005                        .@"volatile" = ptr_info.flags.is_volatile,
 5006                    });
 5007                    try ptr_mat.finish(isel);
 5008                } else {
 5009                    try dst_vi.value.defAddr(isel, .fromInterned(ptr_info.child), .{}) orelse break :unused;
 5010
 5011                    try call.prepareReturn(isel);
 5012                    try call.finishReturn(isel);
 5013
 5014                    try call.prepareCallee(isel);
 5015                    try isel.global_relocs.append(gpa, .{
 5016                        .name = "memcpy",
 5017                        .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 5018                    });
 5019                    try isel.emit(.bl(0));
 5020                    try call.finishCallee(isel);
 5021
 5022                    try call.prepareParams(isel);
 5023                    const ptr_vi = try isel.use(ty_op.operand);
 5024                    try isel.movImmediate(.x2, size);
 5025                    try call.paramLiveOut(isel, ptr_vi, .r1);
 5026                    try call.paramAddress(isel, dst_vi.value, .r0);
 5027                    try call.finishParams(isel);
 5028                }
 5029            }
 5030
 5031            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5032        },
 5033        .ret, .ret_safe => {
 5034            assert(isel.blocks.keys()[0] == Block.main);
 5035            try isel.blocks.values()[0].branch(isel);
 5036            if (isel.live_values.get(Block.main)) |ret_vi| {
 5037                const un_op = air.data(air.inst_index).un_op;
 5038                const src_vi = try isel.use(un_op);
 5039                switch (ret_vi.parent(isel)) {
 5040                    .unallocated, .stack_slot => if (ret_vi.hint(isel)) |ret_ra| {
 5041                        try src_vi.liveOut(isel, ret_ra);
 5042                    } else {
 5043                        var ret_part_it = ret_vi.parts(isel);
 5044                        var src_part_it = src_vi.parts(isel);
 5045                        if (src_part_it.only()) |_| {
 5046                            try isel.values.ensureUnusedCapacity(gpa, ret_part_it.remaining);
 5047                            src_vi.setParts(isel, ret_part_it.remaining);
 5048                            while (ret_part_it.next()) |ret_part_vi| {
 5049                                const src_part_vi = src_vi.addPart(
 5050                                    isel,
 5051                                    ret_part_vi.get(isel).offset_from_parent,
 5052                                    ret_part_vi.size(isel),
 5053                                );
 5054                                switch (ret_part_vi.signedness(isel)) {
 5055                                    .signed => src_part_vi.setSignedness(isel, .signed),
 5056                                    .unsigned => {},
 5057                                }
 5058                                if (ret_part_vi.isVector(isel)) src_part_vi.setIsVector(isel);
 5059                            }
 5060                            ret_part_it = ret_vi.parts(isel);
 5061                            src_part_it = src_vi.parts(isel);
 5062                        }
 5063                        while (ret_part_it.next()) |ret_part_vi| {
 5064                            const src_part_vi = src_part_it.next().?;
 5065                            assert(ret_part_vi.get(isel).offset_from_parent == src_part_vi.get(isel).offset_from_parent);
 5066                            assert(ret_part_vi.size(isel) == src_part_vi.size(isel));
 5067                            try src_part_vi.liveOut(isel, ret_part_vi.hint(isel).?);
 5068                        }
 5069                    },
 5070                    .value, .constant => unreachable,
 5071                    .address => |address_vi| {
 5072                        const ptr_mat = try address_vi.matReg(isel);
 5073                        try src_vi.store(isel, isel.air.typeOf(un_op, ip), ptr_mat.ra, .{});
 5074                        try ptr_mat.finish(isel);
 5075                    },
 5076                }
 5077            }
 5078            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5079        },
 5080        .ret_load => {
 5081            const un_op = air.data(air.inst_index).un_op;
 5082            const ptr_ty = isel.air.typeOf(un_op, ip);
 5083            const ptr_info = ptr_ty.ptrInfo(zcu);
 5084            if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed load", .{});
 5085
 5086            assert(isel.blocks.keys()[0] == Block.main);
 5087            try isel.blocks.values()[0].branch(isel);
 5088            if (isel.live_values.get(Block.main)) |ret_vi| switch (ret_vi.parent(isel)) {
 5089                .unallocated, .stack_slot => {
 5090                    var ret_part_it: Value.PartIterator = if (ret_vi.hint(isel)) |_| .initOne(ret_vi) else ret_vi.parts(isel);
 5091                    while (ret_part_it.next()) |ret_part_vi| try ret_part_vi.liveOut(isel, ret_part_vi.hint(isel).?);
 5092                    const ptr_vi = try isel.use(un_op);
 5093                    const ptr_mat = try ptr_vi.matReg(isel);
 5094                    _ = try ret_vi.load(isel, .fromInterned(ptr_info.child), ptr_mat.ra, .{});
 5095                    try ptr_mat.finish(isel);
 5096                },
 5097                .value, .constant => unreachable,
 5098                .address => {},
 5099            };
 5100            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5101        },
 5102        .store, .store_safe, .atomic_store_unordered => {
 5103            const bin_op = air.data(air.inst_index).bin_op;
 5104            const ptr_ty = isel.air.typeOf(bin_op.lhs, ip);
 5105            const ptr_info = ptr_ty.ptrInfo(zcu);
 5106            if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed store", .{});
 5107            if (bin_op.rhs.toInterned()) |rhs_val| if (ip.isUndef(rhs_val))
 5108                break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5109
 5110            const src_vi = try isel.use(bin_op.rhs);
 5111            const size = src_vi.size(isel);
 5112            if (ZigType.fromInterned(ptr_info.child).zigTypeTag(zcu) != .@"union") switch (size) {
 5113                0 => unreachable,
 5114                1...Value.max_parts => {
 5115                    const ptr_vi = try isel.use(bin_op.lhs);
 5116                    const ptr_mat = try ptr_vi.matReg(isel);
 5117                    try src_vi.store(isel, isel.air.typeOf(bin_op.rhs, ip), ptr_mat.ra, .{
 5118                        .@"volatile" = ptr_info.flags.is_volatile,
 5119                    });
 5120                    try ptr_mat.finish(isel);
 5121
 5122                    break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5123                },
 5124                else => {},
 5125            };
 5126            try call.prepareReturn(isel);
 5127            try call.finishReturn(isel);
 5128
 5129            try call.prepareCallee(isel);
 5130            try isel.global_relocs.append(gpa, .{
 5131                .name = "memcpy",
 5132                .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 5133            });
 5134            try isel.emit(.bl(0));
 5135            try call.finishCallee(isel);
 5136
 5137            try call.prepareParams(isel);
 5138            const ptr_vi = try isel.use(bin_op.lhs);
 5139            try isel.movImmediate(.x2, size);
 5140            try call.paramAddress(isel, src_vi, .r1);
 5141            try call.paramLiveOut(isel, ptr_vi, .r0);
 5142            try call.finishParams(isel);
 5143
 5144            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5145        },
 5146        .unreach => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
 5147        .fptrunc, .fpext => {
 5148            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5149                defer dst_vi.value.deref(isel);
 5150
 5151                const ty_op = air.data(air.inst_index).ty_op;
 5152                const dst_ty = ty_op.ty.toType();
 5153                const dst_bits = dst_ty.floatBits(isel.target);
 5154                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 5155                const src_bits = src_ty.floatBits(isel.target);
 5156                assert(dst_bits != src_bits);
 5157                switch (@max(dst_bits, src_bits)) {
 5158                    else => unreachable,
 5159                    16, 32, 64 => {
 5160                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5161                        const src_vi = try isel.use(ty_op.operand);
 5162                        const src_mat = try src_vi.matReg(isel);
 5163                        try isel.emit(.fcvt(switch (dst_bits) {
 5164                            else => unreachable,
 5165                            16 => dst_ra.h(),
 5166                            32 => dst_ra.s(),
 5167                            64 => dst_ra.d(),
 5168                        }, switch (src_bits) {
 5169                            else => unreachable,
 5170                            16 => src_mat.ra.h(),
 5171                            32 => src_mat.ra.s(),
 5172                            64 => src_mat.ra.d(),
 5173                        }));
 5174                        try src_mat.finish(isel);
 5175                    },
 5176                    80, 128 => {
 5177                        try call.prepareReturn(isel);
 5178                        switch (dst_bits) {
 5179                            else => unreachable,
 5180                            16, 32, 64, 128 => try call.returnLiveIn(isel, dst_vi.value, .v0),
 5181                            80 => {
 5182                                var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
 5183                                const dst_hi16_vi = try dst_hi16_it.only(isel);
 5184                                try call.returnLiveIn(isel, dst_hi16_vi.?, .r1);
 5185                                var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 5186                                const dst_lo64_vi = try dst_lo64_it.only(isel);
 5187                                try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
 5188                            },
 5189                        }
 5190                        try call.finishReturn(isel);
 5191
 5192                        try call.prepareCallee(isel);
 5193                        try isel.global_relocs.append(gpa, .{
 5194                            .name = switch (dst_bits) {
 5195                                else => unreachable,
 5196                                16 => switch (src_bits) {
 5197                                    else => unreachable,
 5198                                    32 => "__truncsfhf2",
 5199                                    64 => "__truncdfhf2",
 5200                                    80 => "__truncxfhf2",
 5201                                    128 => "__trunctfhf2",
 5202                                },
 5203                                32 => switch (src_bits) {
 5204                                    else => unreachable,
 5205                                    16 => "__extendhfsf2",
 5206                                    64 => "__truncdfsf2",
 5207                                    80 => "__truncxfsf2",
 5208                                    128 => "__trunctfsf2",
 5209                                },
 5210                                64 => switch (src_bits) {
 5211                                    else => unreachable,
 5212                                    16 => "__extendhfdf2",
 5213                                    32 => "__extendsfdf2",
 5214                                    80 => "__truncxfdf2",
 5215                                    128 => "__trunctfdf2",
 5216                                },
 5217                                80 => switch (src_bits) {
 5218                                    else => unreachable,
 5219                                    16 => "__extendhfxf2",
 5220                                    32 => "__extendsfxf2",
 5221                                    64 => "__extenddfxf2",
 5222                                    128 => "__trunctfxf2",
 5223                                },
 5224                                128 => switch (src_bits) {
 5225                                    else => unreachable,
 5226                                    16 => "__extendhftf2",
 5227                                    32 => "__extendsftf2",
 5228                                    64 => "__extenddftf2",
 5229                                    80 => "__extendxftf2",
 5230                                },
 5231                            },
 5232                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 5233                        });
 5234                        try isel.emit(.bl(0));
 5235                        try call.finishCallee(isel);
 5236
 5237                        try call.prepareParams(isel);
 5238                        const src_vi = try isel.use(ty_op.operand);
 5239                        switch (src_bits) {
 5240                            else => unreachable,
 5241                            16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
 5242                            80 => {
 5243                                var src_hi16_it = src_vi.field(src_ty, 8, 8);
 5244                                const src_hi16_vi = try src_hi16_it.only(isel);
 5245                                try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
 5246                                var src_lo64_it = src_vi.field(src_ty, 0, 8);
 5247                                const src_lo64_vi = try src_lo64_it.only(isel);
 5248                                try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
 5249                            },
 5250                        }
 5251                        try call.finishParams(isel);
 5252                    },
 5253                }
 5254            }
 5255            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5256        },
 5257        .intcast => |air_tag| {
 5258            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5259                defer dst_vi.value.deref(isel);
 5260
 5261                const ty_op = air.data(air.inst_index).ty_op;
 5262                const dst_ty = ty_op.ty.toType();
 5263                const dst_int_info = dst_ty.intInfo(zcu);
 5264                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 5265                const src_int_info = src_ty.intInfo(zcu);
 5266                const can_be_negative = dst_int_info.signedness == .signed and
 5267                    src_int_info.signedness == .signed;
 5268                if ((dst_int_info.bits <= 8 and src_int_info.bits <= 8) or
 5269                    (dst_int_info.bits > 8 and dst_int_info.bits <= 16 and
 5270                        src_int_info.bits > 8 and src_int_info.bits <= 16) or
 5271                    (dst_int_info.bits > 16 and dst_int_info.bits <= 32 and
 5272                        src_int_info.bits > 16 and src_int_info.bits <= 32) or
 5273                    (dst_int_info.bits > 32 and dst_int_info.bits <= 64 and
 5274                        src_int_info.bits > 32 and src_int_info.bits <= 64) or
 5275                    (dst_int_info.bits > 64 and src_int_info.bits > 64 and
 5276                        (dst_int_info.bits - 1) / 128 == (src_int_info.bits - 1) / 128))
 5277                {
 5278                    try dst_vi.value.move(isel, ty_op.operand);
 5279                } else if (dst_int_info.bits <= 32 and src_int_info.bits <= 64) {
 5280                    const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5281                    const src_vi = try isel.use(ty_op.operand);
 5282                    const src_mat = try src_vi.matReg(isel);
 5283                    try isel.emit(.orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
 5284                    try src_mat.finish(isel);
 5285                } else if (dst_int_info.bits <= 64 and src_int_info.bits <= 32) {
 5286                    const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5287                    const src_vi = try isel.use(ty_op.operand);
 5288                    const src_mat = try src_vi.matReg(isel);
 5289                    try isel.emit(if (can_be_negative) .sbfm(dst_ra.x(), src_mat.ra.x(), .{
 5290                        .N = .doubleword,
 5291                        .immr = 0,
 5292                        .imms = @intCast(src_int_info.bits - 1),
 5293                    }) else .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
 5294                    try src_mat.finish(isel);
 5295                } else if (dst_int_info.bits <= 32 and src_int_info.bits <= 128) {
 5296                    assert(src_int_info.bits > 64);
 5297                    const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5298                    const src_vi = try isel.use(ty_op.operand);
 5299
 5300                    var src_lo64_it = src_vi.field(src_ty, 0, 8);
 5301                    const src_lo64_vi = try src_lo64_it.only(isel);
 5302                    const src_lo64_mat = try src_lo64_vi.?.matReg(isel);
 5303                    try isel.emit(.orr(dst_ra.w(), .wzr, .{ .register = src_lo64_mat.ra.w() }));
 5304                    try src_lo64_mat.finish(isel);
 5305                } else if (dst_int_info.bits <= 64 and src_int_info.bits <= 128) {
 5306                    assert(dst_int_info.bits > 32 and src_int_info.bits > 64);
 5307                    const src_vi = try isel.use(ty_op.operand);
 5308
 5309                    var src_lo64_it = src_vi.field(src_ty, 0, 8);
 5310                    const src_lo64_vi = try src_lo64_it.only(isel);
 5311                    try dst_vi.value.copy(isel, dst_ty, src_lo64_vi.?);
 5312                } else if (dst_int_info.bits <= 128 and src_int_info.bits <= 64) {
 5313                    assert(dst_int_info.bits > 64);
 5314                    const src_vi = try isel.use(ty_op.operand);
 5315
 5316                    var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 5317                    const dst_lo64_vi = try dst_lo64_it.only(isel);
 5318                    if (src_int_info.bits <= 32) unused_lo64: {
 5319                        const dst_lo64_ra = try dst_lo64_vi.?.defReg(isel) orelse break :unused_lo64;
 5320                        const src_mat = try src_vi.matReg(isel);
 5321                        try isel.emit(if (can_be_negative) .sbfm(dst_lo64_ra.x(), src_mat.ra.x(), .{
 5322                            .N = .doubleword,
 5323                            .immr = 0,
 5324                            .imms = @intCast(src_int_info.bits - 1),
 5325                        }) else .orr(dst_lo64_ra.w(), .wzr, .{ .register = src_mat.ra.w() }));
 5326                        try src_mat.finish(isel);
 5327                    } else try dst_lo64_vi.?.copy(isel, src_ty, src_vi);
 5328
 5329                    var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
 5330                    const dst_hi64_vi = try dst_hi64_it.only(isel);
 5331                    const dst_hi64_ra = try dst_hi64_vi.?.defReg(isel);
 5332                    if (dst_hi64_ra) |dst_ra| switch (can_be_negative) {
 5333                        false => try isel.emit(.orr(dst_ra.x(), .xzr, .{ .register = .xzr })),
 5334                        true => {
 5335                            const src_mat = try src_vi.matReg(isel);
 5336                            try isel.emit(.sbfm(dst_ra.x(), src_mat.ra.x(), .{
 5337                                .N = .doubleword,
 5338                                .immr = @intCast(src_int_info.bits - 1),
 5339                                .imms = @intCast(src_int_info.bits - 1),
 5340                            }));
 5341                            try src_mat.finish(isel);
 5342                        },
 5343                    };
 5344                } else return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 5345            }
 5346            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5347        },
 5348        .intcast_safe => |air_tag| {
 5349            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5350                defer dst_vi.value.deref(isel);
 5351
 5352                const ty_op = air.data(air.inst_index).ty_op;
 5353                const dst_ty = ty_op.ty.toType();
 5354                const dst_int_info = dst_ty.intInfo(zcu);
 5355                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 5356                const src_int_info = src_ty.intInfo(zcu);
 5357                const can_be_negative = dst_int_info.signedness == .signed and
 5358                    src_int_info.signedness == .signed;
 5359                const panic_id: Zcu.SimplePanicId = panic_id: switch (dst_ty.zigTypeTag(zcu)) {
 5360                    else => unreachable,
 5361                    .int => .integer_out_of_bounds,
 5362                    .@"enum" => {
 5363                        if (!dst_ty.isNonexhaustiveEnum(zcu)) {
 5364                            return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 5365                        }
 5366                        break :panic_id .invalid_enum_value;
 5367                    },
 5368                };
 5369                if (dst_ty.toIntern() == src_ty.toIntern()) {
 5370                    try dst_vi.value.move(isel, ty_op.operand);
 5371                } else if (dst_int_info.bits <= 64 and src_int_info.bits <= 64) {
 5372                    const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5373                    const src_vi = try isel.use(ty_op.operand);
 5374                    const dst_active_bits = dst_int_info.bits - @intFromBool(dst_int_info.signedness == .signed);
 5375                    const src_active_bits = src_int_info.bits - @intFromBool(src_int_info.signedness == .signed);
 5376                    if ((dst_int_info.signedness != .unsigned or src_int_info.signedness != .signed) and dst_active_bits >= src_active_bits) {
 5377                        const src_mat = try src_vi.matReg(isel);
 5378                        try isel.emit(if (can_be_negative and dst_active_bits > 32 and src_active_bits <= 32)
 5379                            .sbfm(dst_ra.x(), src_mat.ra.x(), .{
 5380                                .N = .doubleword,
 5381                                .immr = 0,
 5382                                .imms = @intCast(src_int_info.bits - 1),
 5383                            })
 5384                        else switch (src_int_info.bits) {
 5385                            else => unreachable,
 5386                            1...32 => .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }),
 5387                            33...64 => .orr(dst_ra.x(), .xzr, .{ .register = src_mat.ra.x() }),
 5388                        });
 5389                        try src_mat.finish(isel);
 5390                    } else {
 5391                        const skip_label = isel.instructions.items.len;
 5392                        try isel.emitPanic(panic_id);
 5393                        try isel.emit(.@"b."(
 5394                            .eq,
 5395                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 5396                        ));
 5397                        if (can_be_negative) {
 5398                            const src_mat = src_mat: {
 5399                                const dst_lock = isel.lockReg(dst_ra);
 5400                                defer dst_lock.unlock(isel);
 5401                                break :src_mat try src_vi.matReg(isel);
 5402                            };
 5403                            try isel.emit(switch (src_int_info.bits) {
 5404                                else => unreachable,
 5405                                1...32 => .subs(.wzr, dst_ra.w(), .{ .register = src_mat.ra.w() }),
 5406                                33...64 => .subs(.xzr, dst_ra.x(), .{ .register = src_mat.ra.x() }),
 5407                            });
 5408                            try isel.emit(switch (@max(dst_int_info.bits, src_int_info.bits)) {
 5409                                else => unreachable,
 5410                                1...32 => .sbfm(dst_ra.w(), src_mat.ra.w(), .{
 5411                                    .N = .word,
 5412                                    .immr = 0,
 5413                                    .imms = @intCast(dst_int_info.bits - 1),
 5414                                }),
 5415                                33...64 => .sbfm(dst_ra.x(), src_mat.ra.x(), .{
 5416                                    .N = .doubleword,
 5417                                    .immr = 0,
 5418                                    .imms = @intCast(dst_int_info.bits - 1),
 5419                                }),
 5420                            });
 5421                            try src_mat.finish(isel);
 5422                        } else {
 5423                            const src_mat = try src_vi.matReg(isel);
 5424                            try isel.emit(switch (@min(dst_int_info.bits, src_int_info.bits)) {
 5425                                else => unreachable,
 5426                                1...32 => .orr(dst_ra.w(), .wzr, .{ .register = src_mat.ra.w() }),
 5427                                33...64 => .orr(dst_ra.x(), .xzr, .{ .register = src_mat.ra.x() }),
 5428                            });
 5429                            const active_bits = @min(dst_active_bits, src_active_bits);
 5430                            try isel.emit(switch (src_int_info.bits) {
 5431                                else => unreachable,
 5432                                1...32 => .ands(.wzr, src_mat.ra.w(), .{ .immediate = .{
 5433                                    .N = .word,
 5434                                    .immr = @intCast(32 - active_bits),
 5435                                    .imms = @intCast(32 - active_bits - 1),
 5436                                } }),
 5437                                33...64 => .ands(.xzr, src_mat.ra.x(), .{ .immediate = .{
 5438                                    .N = .doubleword,
 5439                                    .immr = @intCast(64 - active_bits),
 5440                                    .imms = @intCast(64 - active_bits - 1),
 5441                                } }),
 5442                            });
 5443                            try src_mat.finish(isel);
 5444                        }
 5445                    }
 5446                } else return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 5447            }
 5448            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5449        },
 5450        .trunc => |air_tag| {
 5451            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5452                defer dst_vi.value.deref(isel);
 5453
 5454                const ty_op = air.data(air.inst_index).ty_op;
 5455                const dst_ty = ty_op.ty.toType();
 5456                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 5457                if (!dst_ty.isAbiInt(zcu) or !src_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 5458                const dst_int_info = dst_ty.intInfo(zcu);
 5459                switch (dst_int_info.bits) {
 5460                    0 => unreachable,
 5461                    1...64 => |dst_bits| {
 5462                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5463                        const src_vi = try isel.use(ty_op.operand);
 5464                        var src_part_it = src_vi.field(src_ty, 0, @min(src_vi.size(isel), 8));
 5465                        const src_part_vi = try src_part_it.only(isel);
 5466                        const src_part_mat = try src_part_vi.?.matReg(isel);
 5467                        try isel.emit(switch (dst_bits) {
 5468                            else => unreachable,
 5469                            1...31 => |bits| switch (dst_int_info.signedness) {
 5470                                .signed => .sbfm(dst_ra.w(), src_part_mat.ra.w(), .{
 5471                                    .N = .word,
 5472                                    .immr = 0,
 5473                                    .imms = @intCast(bits - 1),
 5474                                }),
 5475                                .unsigned => .ubfm(dst_ra.w(), src_part_mat.ra.w(), .{
 5476                                    .N = .word,
 5477                                    .immr = 0,
 5478                                    .imms = @intCast(bits - 1),
 5479                                }),
 5480                            },
 5481                            32 => .orr(dst_ra.w(), .wzr, .{ .register = src_part_mat.ra.w() }),
 5482                            33...63 => |bits| switch (dst_int_info.signedness) {
 5483                                .signed => .sbfm(dst_ra.x(), src_part_mat.ra.x(), .{
 5484                                    .N = .doubleword,
 5485                                    .immr = 0,
 5486                                    .imms = @intCast(bits - 1),
 5487                                }),
 5488                                .unsigned => .ubfm(dst_ra.x(), src_part_mat.ra.x(), .{
 5489                                    .N = .doubleword,
 5490                                    .immr = 0,
 5491                                    .imms = @intCast(bits - 1),
 5492                                }),
 5493                            },
 5494                            64 => .orr(dst_ra.x(), .xzr, .{ .register = src_part_mat.ra.x() }),
 5495                        });
 5496                        try src_part_mat.finish(isel);
 5497                    },
 5498                    65...128 => |dst_bits| switch (src_ty.intInfo(zcu).bits) {
 5499                        0 => unreachable,
 5500                        65...128 => {
 5501                            const src_vi = try isel.use(ty_op.operand);
 5502                            var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
 5503                            const dst_hi64_vi = try dst_hi64_it.only(isel);
 5504                            if (try dst_hi64_vi.?.defReg(isel)) |dst_hi64_ra| {
 5505                                var src_hi64_it = src_vi.field(src_ty, 8, 8);
 5506                                const src_hi64_vi = try src_hi64_it.only(isel);
 5507                                const src_hi64_mat = try src_hi64_vi.?.matReg(isel);
 5508                                try isel.emit(switch (dst_int_info.signedness) {
 5509                                    .signed => .sbfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
 5510                                        .N = .doubleword,
 5511                                        .immr = 0,
 5512                                        .imms = @intCast(dst_bits - 64 - 1),
 5513                                    }),
 5514                                    .unsigned => .ubfm(dst_hi64_ra.x(), src_hi64_mat.ra.x(), .{
 5515                                        .N = .doubleword,
 5516                                        .immr = 0,
 5517                                        .imms = @intCast(dst_bits - 64 - 1),
 5518                                    }),
 5519                                });
 5520                                try src_hi64_mat.finish(isel);
 5521                            }
 5522                            var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 5523                            const dst_lo64_vi = try dst_lo64_it.only(isel);
 5524                            if (try dst_lo64_vi.?.defReg(isel)) |dst_lo64_ra| {
 5525                                var src_lo64_it = src_vi.field(src_ty, 0, 8);
 5526                                const src_lo64_vi = try src_lo64_it.only(isel);
 5527                                try src_lo64_vi.?.liveOut(isel, dst_lo64_ra);
 5528                            }
 5529                        },
 5530                        else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
 5531                    },
 5532                    else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
 5533                }
 5534            }
 5535            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5536        },
 5537        .optional_payload => {
 5538            if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| unused: {
 5539                defer payload_vi.value.deref(isel);
 5540
 5541                const ty_op = air.data(air.inst_index).ty_op;
 5542                const opt_ty = isel.air.typeOf(ty_op.operand, ip);
 5543                if (opt_ty.optionalReprIsPayload(zcu)) {
 5544                    try payload_vi.value.move(isel, ty_op.operand);
 5545                    break :unused;
 5546                }
 5547
 5548                const opt_vi = try isel.use(ty_op.operand);
 5549                var payload_part_it = opt_vi.field(opt_ty, 0, payload_vi.value.size(isel));
 5550                const payload_part_vi = try payload_part_it.only(isel);
 5551                try payload_vi.value.copy(isel, ty_op.ty.toType(), payload_part_vi.?);
 5552            }
 5553            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5554        },
 5555        .optional_payload_ptr => {
 5556            if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| {
 5557                defer payload_ptr_vi.value.deref(isel);
 5558                const ty_op = air.data(air.inst_index).ty_op;
 5559                try payload_ptr_vi.value.move(isel, ty_op.operand);
 5560            }
 5561            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5562        },
 5563        .optional_payload_ptr_set => {
 5564            if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| {
 5565                defer payload_ptr_vi.value.deref(isel);
 5566                const ty_op = air.data(air.inst_index).ty_op;
 5567                const opt_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu);
 5568                if (!opt_ty.optionalReprIsPayload(zcu)) {
 5569                    const opt_ptr_vi = try isel.use(ty_op.operand);
 5570                    const opt_ptr_mat = try opt_ptr_vi.matReg(isel);
 5571                    const has_value_ra = try isel.allocIntReg();
 5572                    defer isel.freeReg(has_value_ra);
 5573                    try isel.storeReg(
 5574                        has_value_ra,
 5575                        1,
 5576                        opt_ptr_mat.ra,
 5577                        opt_ty.optionalChild(zcu).abiSize(zcu),
 5578                    );
 5579                    try opt_ptr_mat.finish(isel);
 5580                    try isel.emit(.movz(has_value_ra.w(), 1, .{ .lsl = .@"0" }));
 5581                }
 5582                try payload_ptr_vi.value.move(isel, ty_op.operand);
 5583            }
 5584            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5585        },
 5586        .wrap_optional => {
 5587            if (isel.live_values.fetchRemove(air.inst_index)) |opt_vi| unused: {
 5588                defer opt_vi.value.deref(isel);
 5589
 5590                const ty_op = air.data(air.inst_index).ty_op;
 5591                if (ty_op.ty.toType().optionalReprIsPayload(zcu)) {
 5592                    try opt_vi.value.move(isel, ty_op.operand);
 5593                    break :unused;
 5594                }
 5595
 5596                const payload_size = isel.air.typeOf(ty_op.operand, ip).abiSize(zcu);
 5597                var payload_part_it = opt_vi.value.field(ty_op.ty.toType(), 0, payload_size);
 5598                const payload_part_vi = try payload_part_it.only(isel);
 5599                try payload_part_vi.?.move(isel, ty_op.operand);
 5600                var has_value_part_it = opt_vi.value.field(ty_op.ty.toType(), payload_size, 1);
 5601                const has_value_part_vi = try has_value_part_it.only(isel);
 5602                const has_value_part_ra = try has_value_part_vi.?.defReg(isel) orelse break :unused;
 5603                try isel.emit(.movz(has_value_part_ra.w(), 1, .{ .lsl = .@"0" }));
 5604            }
 5605            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5606        },
 5607        .unwrap_errunion_payload => {
 5608            if (isel.live_values.fetchRemove(air.inst_index)) |payload_vi| {
 5609                defer payload_vi.value.deref(isel);
 5610
 5611                const ty_op = air.data(air.inst_index).ty_op;
 5612                const error_union_ty = isel.air.typeOf(ty_op.operand, ip);
 5613
 5614                const error_union_vi = try isel.use(ty_op.operand);
 5615                var payload_part_it = error_union_vi.field(
 5616                    error_union_ty,
 5617                    codegen.errUnionPayloadOffset(ty_op.ty.toType(), zcu),
 5618                    payload_vi.value.size(isel),
 5619                );
 5620                const payload_part_vi = try payload_part_it.only(isel);
 5621                try payload_vi.value.copy(isel, ty_op.ty.toType(), payload_part_vi.?);
 5622            }
 5623            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5624        },
 5625        .unwrap_errunion_err => {
 5626            if (isel.live_values.fetchRemove(air.inst_index)) |error_set_vi| {
 5627                defer error_set_vi.value.deref(isel);
 5628
 5629                const ty_op = air.data(air.inst_index).ty_op;
 5630                const error_union_ty = isel.air.typeOf(ty_op.operand, ip);
 5631
 5632                const error_union_vi = try isel.use(ty_op.operand);
 5633                var error_set_part_it = error_union_vi.field(
 5634                    error_union_ty,
 5635                    codegen.errUnionErrorOffset(error_union_ty.errorUnionPayload(zcu), zcu),
 5636                    error_set_vi.value.size(isel),
 5637                );
 5638                const error_set_part_vi = try error_set_part_it.only(isel);
 5639                try error_set_vi.value.copy(isel, ty_op.ty.toType(), error_set_part_vi.?);
 5640            }
 5641            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5642        },
 5643        .unwrap_errunion_payload_ptr => {
 5644            if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
 5645                defer payload_ptr_vi.value.deref(isel);
 5646                const ty_op = air.data(air.inst_index).ty_op;
 5647                switch (codegen.errUnionPayloadOffset(ty_op.ty.toType().childType(zcu), zcu)) {
 5648                    0 => try payload_ptr_vi.value.move(isel, ty_op.operand),
 5649                    else => |payload_offset| {
 5650                        const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
 5651                        const error_union_ptr_vi = try isel.use(ty_op.operand);
 5652                        const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
 5653                        const lo12: u12 = @truncate(payload_offset >> 0);
 5654                        const hi12: u12 = @intCast(payload_offset >> 12);
 5655                        if (hi12 > 0) try isel.emit(.add(
 5656                            payload_ptr_ra.x(),
 5657                            if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
 5658                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 5659                        ));
 5660                        if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
 5661                        try error_union_ptr_mat.finish(isel);
 5662                    },
 5663                }
 5664            }
 5665            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5666        },
 5667        .unwrap_errunion_err_ptr => {
 5668            if (isel.live_values.fetchRemove(air.inst_index)) |error_vi| {
 5669                defer error_vi.value.deref(isel);
 5670                const ty_op = air.data(air.inst_index).ty_op;
 5671                const error_union_ptr_ty = isel.air.typeOf(ty_op.operand, ip);
 5672                const error_union_ptr_info = error_union_ptr_ty.ptrInfo(zcu);
 5673                const error_union_ptr_vi = try isel.use(ty_op.operand);
 5674                const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
 5675                _ = try error_vi.value.load(isel, ty_op.ty.toType(), error_union_ptr_mat.ra, .{
 5676                    .offset = codegen.errUnionErrorOffset(
 5677                        ZigType.fromInterned(error_union_ptr_info.child).errorUnionPayload(zcu),
 5678                        zcu,
 5679                    ),
 5680                    .@"volatile" = error_union_ptr_info.flags.is_volatile,
 5681                });
 5682                try error_union_ptr_mat.finish(isel);
 5683            }
 5684            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5685        },
 5686        .errunion_payload_ptr_set => {
 5687            if (isel.live_values.fetchRemove(air.inst_index)) |payload_ptr_vi| unused: {
 5688                defer payload_ptr_vi.value.deref(isel);
 5689                const ty_op = air.data(air.inst_index).ty_op;
 5690                const payload_ty = ty_op.ty.toType().childType(zcu);
 5691                const error_union_ty = isel.air.typeOf(ty_op.operand, ip).childType(zcu);
 5692                const error_set_size = error_union_ty.errorUnionSet(zcu).abiSize(zcu);
 5693                const error_union_ptr_vi = try isel.use(ty_op.operand);
 5694                const error_union_ptr_mat = try error_union_ptr_vi.matReg(isel);
 5695                if (error_set_size > 0) try isel.storeReg(
 5696                    .zr,
 5697                    error_set_size,
 5698                    error_union_ptr_mat.ra,
 5699                    codegen.errUnionErrorOffset(payload_ty, zcu),
 5700                );
 5701                switch (codegen.errUnionPayloadOffset(payload_ty, zcu)) {
 5702                    0 => {
 5703                        try error_union_ptr_mat.finish(isel);
 5704                        try payload_ptr_vi.value.move(isel, ty_op.operand);
 5705                    },
 5706                    else => |payload_offset| {
 5707                        const payload_ptr_ra = try payload_ptr_vi.value.defReg(isel) orelse break :unused;
 5708                        const lo12: u12 = @truncate(payload_offset >> 0);
 5709                        const hi12: u12 = @intCast(payload_offset >> 12);
 5710                        if (hi12 > 0) try isel.emit(.add(
 5711                            payload_ptr_ra.x(),
 5712                            if (lo12 > 0) payload_ptr_ra.x() else error_union_ptr_mat.ra.x(),
 5713                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 5714                        ));
 5715                        if (lo12 > 0) try isel.emit(.add(payload_ptr_ra.x(), error_union_ptr_mat.ra.x(), .{ .immediate = lo12 }));
 5716                        try error_union_ptr_mat.finish(isel);
 5717                    },
 5718                }
 5719            }
 5720            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5721        },
 5722        .wrap_errunion_payload => {
 5723            if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
 5724                defer error_union_vi.value.deref(isel);
 5725
 5726                const ty_op = air.data(air.inst_index).ty_op;
 5727                const error_union_ty = ty_op.ty.toType();
 5728                const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
 5729                const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
 5730                const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
 5731                const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
 5732                const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
 5733                const error_set_size = error_set_ty.abiSize(zcu);
 5734                const payload_size = payload_ty.abiSize(zcu);
 5735
 5736                var payload_part_it = error_union_vi.value.field(error_union_ty, payload_offset, payload_size);
 5737                const payload_part_vi = try payload_part_it.only(isel);
 5738                try payload_part_vi.?.move(isel, ty_op.operand);
 5739                var error_set_part_it = error_union_vi.value.field(error_union_ty, error_set_offset, error_set_size);
 5740                const error_set_part_vi = try error_set_part_it.only(isel);
 5741                if (try error_set_part_vi.?.defReg(isel)) |error_set_part_ra| try isel.emit(switch (error_set_size) {
 5742                    else => unreachable,
 5743                    1...4 => .orr(error_set_part_ra.w(), .wzr, .{ .register = .wzr }),
 5744                    5...8 => .orr(error_set_part_ra.x(), .xzr, .{ .register = .xzr }),
 5745                });
 5746            }
 5747            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5748        },
 5749        .wrap_errunion_err => {
 5750            if (isel.live_values.fetchRemove(air.inst_index)) |error_union_vi| {
 5751                defer error_union_vi.value.deref(isel);
 5752
 5753                const ty_op = air.data(air.inst_index).ty_op;
 5754                const error_union_ty = ty_op.ty.toType();
 5755                const error_union_info = ip.indexToKey(error_union_ty.toIntern()).error_union_type;
 5756                const error_set_ty: ZigType = .fromInterned(error_union_info.error_set_type);
 5757                const payload_ty: ZigType = .fromInterned(error_union_info.payload_type);
 5758                const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
 5759                const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
 5760                const error_set_size = error_set_ty.abiSize(zcu);
 5761                const payload_size = payload_ty.abiSize(zcu);
 5762
 5763                var error_set_part_it = error_union_vi.value.field(error_union_ty, error_set_offset, error_set_size);
 5764                const error_set_part_vi = try error_set_part_it.only(isel);
 5765                try error_set_part_vi.?.move(isel, ty_op.operand);
 5766                if (payload_size > 0) {
 5767                    var payload_part_it = error_union_vi.value.field(error_union_ty, payload_offset, payload_size);
 5768                    const payload_part_vi = try payload_part_it.only(isel);
 5769                    try payload_part_vi.?.defUndef(isel, payload_ty, .{});
 5770                }
 5771            }
 5772            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5773        },
 5774        .struct_field_ptr => {
 5775            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5776                defer dst_vi.value.deref(isel);
 5777                const ty_pl = air.data(air.inst_index).ty_pl;
 5778                const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
 5779                switch (codegen.fieldOffset(
 5780                    isel.air.typeOf(extra.struct_operand, ip),
 5781                    ty_pl.ty.toType(),
 5782                    extra.field_index,
 5783                    zcu,
 5784                )) {
 5785                    0 => try dst_vi.value.move(isel, extra.struct_operand),
 5786                    else => |field_offset| {
 5787                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5788                        const src_vi = try isel.use(extra.struct_operand);
 5789                        const src_mat = try src_vi.matReg(isel);
 5790                        const lo12: u12 = @truncate(field_offset >> 0);
 5791                        const hi12: u12 = @intCast(field_offset >> 12);
 5792                        if (hi12 > 0) try isel.emit(.add(
 5793                            dst_ra.x(),
 5794                            if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
 5795                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 5796                        ));
 5797                        if (lo12 > 0) try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
 5798                        try src_mat.finish(isel);
 5799                    },
 5800                }
 5801            }
 5802            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5803        },
 5804        .struct_field_ptr_index_0,
 5805        .struct_field_ptr_index_1,
 5806        .struct_field_ptr_index_2,
 5807        .struct_field_ptr_index_3,
 5808        => |air_tag| {
 5809            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5810                defer dst_vi.value.deref(isel);
 5811                const ty_op = air.data(air.inst_index).ty_op;
 5812                switch (codegen.fieldOffset(
 5813                    isel.air.typeOf(ty_op.operand, ip),
 5814                    ty_op.ty.toType(),
 5815                    switch (air_tag) {
 5816                        else => unreachable,
 5817                        .struct_field_ptr_index_0 => 0,
 5818                        .struct_field_ptr_index_1 => 1,
 5819                        .struct_field_ptr_index_2 => 2,
 5820                        .struct_field_ptr_index_3 => 3,
 5821                    },
 5822                    zcu,
 5823                )) {
 5824                    0 => try dst_vi.value.move(isel, ty_op.operand),
 5825                    else => |field_offset| {
 5826                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5827                        const src_vi = try isel.use(ty_op.operand);
 5828                        const src_mat = try src_vi.matReg(isel);
 5829                        const lo12: u12 = @truncate(field_offset >> 0);
 5830                        const hi12: u12 = @intCast(field_offset >> 12);
 5831                        if (hi12 > 0) try isel.emit(.add(
 5832                            dst_ra.x(),
 5833                            if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
 5834                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 5835                        ));
 5836                        if (lo12 > 0) try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
 5837                        try src_mat.finish(isel);
 5838                    },
 5839                }
 5840            }
 5841            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5842        },
 5843        .struct_field_val => {
 5844            if (isel.live_values.fetchRemove(air.inst_index)) |field_vi| unused: {
 5845                defer field_vi.value.deref(isel);
 5846
 5847                const ty_pl = air.data(air.inst_index).ty_pl;
 5848                const extra = isel.air.extraData(Air.StructField, ty_pl.payload).data;
 5849                const agg_ty = isel.air.typeOf(extra.struct_operand, ip);
 5850                const field_ty = ty_pl.ty.toType();
 5851                const field_bit_offset, const field_bit_size, const is_packed = switch (agg_ty.containerLayout(zcu)) {
 5852                    .auto, .@"extern" => .{
 5853                        8 * agg_ty.structFieldOffset(extra.field_index, zcu),
 5854                        8 * field_ty.abiSize(zcu),
 5855                        false,
 5856                    },
 5857                    .@"packed" => .{
 5858                        if (zcu.typeToPackedStruct(agg_ty)) |loaded_struct|
 5859                            zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index)
 5860                        else
 5861                            0,
 5862                        field_ty.bitSize(zcu),
 5863                        true,
 5864                    },
 5865                };
 5866                if (is_packed) return isel.fail("packed field of {f}", .{
 5867                    isel.fmtType(agg_ty),
 5868                });
 5869
 5870                const agg_vi = try isel.use(extra.struct_operand);
 5871                switch (agg_ty.zigTypeTag(zcu)) {
 5872                    else => unreachable,
 5873                    .@"struct" => {
 5874                        var agg_part_it = agg_vi.field(agg_ty, @divExact(field_bit_offset, 8), @divExact(field_bit_size, 8));
 5875                        while (try agg_part_it.next(isel)) |agg_part| {
 5876                            var field_part_it = field_vi.value.field(ty_pl.ty.toType(), agg_part.offset, agg_part.vi.size(isel));
 5877                            const field_part_vi = try field_part_it.only(isel);
 5878                            if (field_part_vi.? == agg_part.vi) continue;
 5879                            var field_subpart_it = field_part_vi.?.parts(isel);
 5880                            const field_part_offset = if (field_subpart_it.only()) |field_subpart_vi|
 5881                                field_subpart_vi.get(isel).offset_from_parent
 5882                            else
 5883                                0;
 5884                            while (field_subpart_it.next()) |field_subpart_vi| {
 5885                                const field_subpart_ra = try field_subpart_vi.defReg(isel) orelse continue;
 5886                                const field_subpart_offset, const field_subpart_size = field_subpart_vi.position(isel);
 5887                                var agg_subpart_it = agg_part.vi.field(
 5888                                    field_ty,
 5889                                    agg_part.offset + field_subpart_offset - field_part_offset,
 5890                                    field_subpart_size,
 5891                                );
 5892                                const agg_subpart_vi = try agg_subpart_it.only(isel);
 5893                                try agg_subpart_vi.?.liveOut(isel, field_subpart_ra);
 5894                            }
 5895                        }
 5896                    },
 5897                    .@"union" => {
 5898                        try field_vi.value.defAddr(isel, field_ty, .{}) orelse break :unused;
 5899
 5900                        try call.prepareReturn(isel);
 5901                        try call.finishReturn(isel);
 5902
 5903                        try call.prepareCallee(isel);
 5904                        try isel.global_relocs.append(gpa, .{
 5905                            .name = "memcpy",
 5906                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 5907                        });
 5908                        try isel.emit(.bl(0));
 5909                        try call.finishCallee(isel);
 5910
 5911                        try call.prepareParams(isel);
 5912                        const union_layout = agg_ty.unionGetLayout(zcu);
 5913                        var payload_it = agg_vi.field(agg_ty, union_layout.payloadOffset(), union_layout.payload_size);
 5914                        const payload_vi = try payload_it.only(isel);
 5915                        try isel.movImmediate(.x2, field_vi.value.size(isel));
 5916                        try call.paramAddress(isel, payload_vi.?, .r1);
 5917                        try call.paramAddress(isel, field_vi.value, .r0);
 5918                        try call.finishParams(isel);
 5919                    },
 5920                }
 5921            }
 5922            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5923        },
 5924        .set_union_tag => {
 5925            const bin_op = air.data(air.inst_index).bin_op;
 5926            const union_ty = isel.air.typeOf(bin_op.lhs, ip).childType(zcu);
 5927            const union_layout = union_ty.unionGetLayout(zcu);
 5928            const tag_vi = try isel.use(bin_op.rhs);
 5929            const union_ptr_vi = try isel.use(bin_op.lhs);
 5930            const union_ptr_mat = try union_ptr_vi.matReg(isel);
 5931            try tag_vi.store(isel, isel.air.typeOf(bin_op.rhs, ip), union_ptr_mat.ra, .{
 5932                .offset = union_layout.tagOffset(),
 5933            });
 5934            try union_ptr_mat.finish(isel);
 5935            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5936        },
 5937        .get_union_tag => {
 5938            if (isel.live_values.fetchRemove(air.inst_index)) |tag_vi| {
 5939                defer tag_vi.value.deref(isel);
 5940                const ty_op = air.data(air.inst_index).ty_op;
 5941                const union_ty = isel.air.typeOf(ty_op.operand, ip);
 5942                const union_layout = union_ty.unionGetLayout(zcu);
 5943                const union_vi = try isel.use(ty_op.operand);
 5944                var tag_part_it = union_vi.field(union_ty, union_layout.tagOffset(), union_layout.tag_size);
 5945                const tag_part_vi = try tag_part_it.only(isel);
 5946                try tag_vi.value.copy(isel, ty_op.ty.toType(), tag_part_vi.?);
 5947            }
 5948            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5949        },
 5950        .slice => {
 5951            if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
 5952                defer slice_vi.value.deref(isel);
 5953                const ty_pl = air.data(air.inst_index).ty_pl;
 5954                const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
 5955                var ptr_part_it = slice_vi.value.field(ty_pl.ty.toType(), 0, 8);
 5956                const ptr_part_vi = try ptr_part_it.only(isel);
 5957                try ptr_part_vi.?.move(isel, bin_op.lhs);
 5958                var len_part_it = slice_vi.value.field(ty_pl.ty.toType(), 8, 8);
 5959                const len_part_vi = try len_part_it.only(isel);
 5960                try len_part_vi.?.move(isel, bin_op.rhs);
 5961            }
 5962            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5963        },
 5964        .slice_len => {
 5965            if (isel.live_values.fetchRemove(air.inst_index)) |len_vi| {
 5966                defer len_vi.value.deref(isel);
 5967                const ty_op = air.data(air.inst_index).ty_op;
 5968                const slice_vi = try isel.use(ty_op.operand);
 5969                var len_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 8, 8);
 5970                const len_part_vi = try len_part_it.only(isel);
 5971                try len_vi.value.copy(isel, ty_op.ty.toType(), len_part_vi.?);
 5972            }
 5973            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5974        },
 5975        .slice_ptr => {
 5976            if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| {
 5977                defer ptr_vi.value.deref(isel);
 5978                const ty_op = air.data(air.inst_index).ty_op;
 5979                const slice_vi = try isel.use(ty_op.operand);
 5980                var ptr_part_it = slice_vi.field(isel.air.typeOf(ty_op.operand, ip), 0, 8);
 5981                const ptr_part_vi = try ptr_part_it.only(isel);
 5982                try ptr_vi.value.copy(isel, ty_op.ty.toType(), ptr_part_vi.?);
 5983            }
 5984            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5985        },
 5986        .ptr_slice_len_ptr => {
 5987            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 5988                defer dst_vi.value.deref(isel);
 5989                const ty_op = air.data(air.inst_index).ty_op;
 5990                const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 5991                const src_vi = try isel.use(ty_op.operand);
 5992                const src_mat = try src_vi.matReg(isel);
 5993                try isel.emit(.add(dst_ra.x(), src_mat.ra.x(), .{ .immediate = 8 }));
 5994                try src_mat.finish(isel);
 5995            }
 5996            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 5997        },
 5998        .ptr_slice_ptr_ptr => {
 5999            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
 6000                defer dst_vi.value.deref(isel);
 6001                const ty_op = air.data(air.inst_index).ty_op;
 6002                try dst_vi.value.move(isel, ty_op.operand);
 6003            }
 6004            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6005        },
 6006        .array_elem_val => {
 6007            if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
 6008                defer elem_vi.value.deref(isel);
 6009
 6010                const bin_op = air.data(air.inst_index).bin_op;
 6011                const array_ty = isel.air.typeOf(bin_op.lhs, ip);
 6012                const elem_ty = array_ty.childType(zcu);
 6013                const elem_size = elem_ty.abiSize(zcu);
 6014                if (elem_size <= 16 and array_ty.arrayLenIncludingSentinel(zcu) <= Value.max_parts) if (bin_op.rhs.toInterned()) |index_val| {
 6015                    const elem_offset = elem_size * Constant.fromInterned(index_val).toUnsignedInt(zcu);
 6016                    const array_vi = try isel.use(bin_op.lhs);
 6017                    var elem_part_it = array_vi.field(array_ty, elem_offset, elem_size);
 6018                    const elem_part_vi = try elem_part_it.only(isel);
 6019                    try elem_vi.value.copy(isel, elem_ty, elem_part_vi.?);
 6020                    break :unused;
 6021                };
 6022                switch (elem_size) {
 6023                    0 => unreachable,
 6024                    1, 2, 4, 8 => {
 6025                        const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
 6026                        const array_ptr_ra = try isel.allocIntReg();
 6027                        defer isel.freeReg(array_ptr_ra);
 6028                        const index_vi = try isel.use(bin_op.rhs);
 6029                        const index_mat = try index_vi.matReg(isel);
 6030                        try isel.emit(switch (elem_size) {
 6031                            else => unreachable,
 6032                            1 => if (elem_vi.value.isVector(isel)) .ldr(elem_ra.b(), .{ .extended_register = .{
 6033                                .base = array_ptr_ra.x(),
 6034                                .index = index_mat.ra.x(),
 6035                                .extend = .{ .lsl = 0 },
 6036                            } }) else switch (elem_vi.value.signedness(isel)) {
 6037                                .signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
 6038                                    .base = array_ptr_ra.x(),
 6039                                    .index = index_mat.ra.x(),
 6040                                    .extend = .{ .lsl = 0 },
 6041                                } }),
 6042                                .unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
 6043                                    .base = array_ptr_ra.x(),
 6044                                    .index = index_mat.ra.x(),
 6045                                    .extend = .{ .lsl = 0 },
 6046                                } }),
 6047                            },
 6048                            2 => if (elem_vi.value.isVector(isel)) .ldr(elem_ra.h(), .{ .extended_register = .{
 6049                                .base = array_ptr_ra.x(),
 6050                                .index = index_mat.ra.x(),
 6051                                .extend = .{ .lsl = 1 },
 6052                            } }) else switch (elem_vi.value.signedness(isel)) {
 6053                                .signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
 6054                                    .base = array_ptr_ra.x(),
 6055                                    .index = index_mat.ra.x(),
 6056                                    .extend = .{ .lsl = 1 },
 6057                                } }),
 6058                                .unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
 6059                                    .base = array_ptr_ra.x(),
 6060                                    .index = index_mat.ra.x(),
 6061                                    .extend = .{ .lsl = 1 },
 6062                                } }),
 6063                            },
 6064                            4 => .ldr(if (elem_vi.value.isVector(isel)) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
 6065                                .base = array_ptr_ra.x(),
 6066                                .index = index_mat.ra.x(),
 6067                                .extend = .{ .lsl = 2 },
 6068                            } }),
 6069                            8 => .ldr(if (elem_vi.value.isVector(isel)) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
 6070                                .base = array_ptr_ra.x(),
 6071                                .index = index_mat.ra.x(),
 6072                                .extend = .{ .lsl = 3 },
 6073                            } }),
 6074                            16 => .ldr(elem_ra.q(), .{ .extended_register = .{
 6075                                .base = array_ptr_ra.x(),
 6076                                .index = index_mat.ra.x(),
 6077                                .extend = .{ .lsl = 4 },
 6078                            } }),
 6079                        });
 6080                        try index_mat.finish(isel);
 6081                        const array_vi = try isel.use(bin_op.lhs);
 6082                        try array_vi.address(isel, 0, array_ptr_ra);
 6083                    },
 6084                    else => {
 6085                        const ptr_ra = try isel.allocIntReg();
 6086                        defer isel.freeReg(ptr_ra);
 6087                        if (!try elem_vi.value.load(isel, elem_ty, ptr_ra, .{})) break :unused;
 6088                        const index_vi = try isel.use(bin_op.rhs);
 6089                        try isel.elemPtr(ptr_ra, ptr_ra, .add, elem_size, index_vi);
 6090                        const array_vi = try isel.use(bin_op.lhs);
 6091                        try array_vi.address(isel, 0, ptr_ra);
 6092                    },
 6093                }
 6094            }
 6095            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6096        },
 6097        .slice_elem_val => {
 6098            if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
 6099                defer elem_vi.value.deref(isel);
 6100
 6101                const bin_op = air.data(air.inst_index).bin_op;
 6102                const slice_ty = isel.air.typeOf(bin_op.lhs, ip);
 6103                const ptr_info = slice_ty.ptrInfo(zcu);
 6104                const elem_size = elem_vi.value.size(isel);
 6105                const elem_is_vector = elem_vi.value.isVector(isel);
 6106                if (switch (elem_size) {
 6107                    0 => unreachable,
 6108                    1, 2, 4, 8 => true,
 6109                    16 => elem_is_vector,
 6110                    else => false,
 6111                }) {
 6112                    const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
 6113                    const slice_vi = try isel.use(bin_op.lhs);
 6114                    const index_vi = try isel.use(bin_op.rhs);
 6115                    var ptr_part_it = slice_vi.field(slice_ty, 0, 8);
 6116                    const ptr_part_vi = try ptr_part_it.only(isel);
 6117                    const base_mat = try ptr_part_vi.?.matReg(isel);
 6118                    const index_mat = try index_vi.matReg(isel);
 6119                    try isel.emit(switch (elem_size) {
 6120                        else => unreachable,
 6121                        1 => if (elem_is_vector) .ldr(elem_ra.b(), .{ .extended_register = .{
 6122                            .base = base_mat.ra.x(),
 6123                            .index = index_mat.ra.x(),
 6124                            .extend = .{ .lsl = 0 },
 6125                        } }) else switch (elem_vi.value.signedness(isel)) {
 6126                            .signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
 6127                                .base = base_mat.ra.x(),
 6128                                .index = index_mat.ra.x(),
 6129                                .extend = .{ .lsl = 0 },
 6130                            } }),
 6131                            .unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
 6132                                .base = base_mat.ra.x(),
 6133                                .index = index_mat.ra.x(),
 6134                                .extend = .{ .lsl = 0 },
 6135                            } }),
 6136                        },
 6137                        2 => if (elem_is_vector) .ldr(elem_ra.h(), .{ .extended_register = .{
 6138                            .base = base_mat.ra.x(),
 6139                            .index = index_mat.ra.x(),
 6140                            .extend = .{ .lsl = 1 },
 6141                        } }) else switch (elem_vi.value.signedness(isel)) {
 6142                            .signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
 6143                                .base = base_mat.ra.x(),
 6144                                .index = index_mat.ra.x(),
 6145                                .extend = .{ .lsl = 1 },
 6146                            } }),
 6147                            .unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
 6148                                .base = base_mat.ra.x(),
 6149                                .index = index_mat.ra.x(),
 6150                                .extend = .{ .lsl = 1 },
 6151                            } }),
 6152                        },
 6153                        4 => .ldr(if (elem_is_vector) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
 6154                            .base = base_mat.ra.x(),
 6155                            .index = index_mat.ra.x(),
 6156                            .extend = .{ .lsl = 2 },
 6157                        } }),
 6158                        8 => .ldr(if (elem_is_vector) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
 6159                            .base = base_mat.ra.x(),
 6160                            .index = index_mat.ra.x(),
 6161                            .extend = .{ .lsl = 3 },
 6162                        } }),
 6163                        16 => if (elem_is_vector) .ldr(elem_ra.q(), .{ .extended_register = .{
 6164                            .base = base_mat.ra.x(),
 6165                            .index = index_mat.ra.x(),
 6166                            .extend = .{ .lsl = 4 },
 6167                        } }) else unreachable,
 6168                    });
 6169                    try index_mat.finish(isel);
 6170                    try base_mat.finish(isel);
 6171                } else {
 6172                    const elem_ptr_ra = try isel.allocIntReg();
 6173                    defer isel.freeReg(elem_ptr_ra);
 6174                    if (!try elem_vi.value.load(isel, slice_ty.elemType2(zcu), elem_ptr_ra, .{
 6175                        .@"volatile" = ptr_info.flags.is_volatile,
 6176                    })) break :unused;
 6177                    const slice_vi = try isel.use(bin_op.lhs);
 6178                    var ptr_part_it = slice_vi.field(slice_ty, 0, 8);
 6179                    const ptr_part_vi = try ptr_part_it.only(isel);
 6180                    const ptr_part_mat = try ptr_part_vi.?.matReg(isel);
 6181                    const index_vi = try isel.use(bin_op.rhs);
 6182                    try isel.elemPtr(elem_ptr_ra, ptr_part_mat.ra, .add, elem_size, index_vi);
 6183                    try ptr_part_mat.finish(isel);
 6184                }
 6185            }
 6186            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6187        },
 6188        .slice_elem_ptr => {
 6189            if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
 6190                defer elem_ptr_vi.value.deref(isel);
 6191                const elem_ptr_ra = try elem_ptr_vi.value.defReg(isel) orelse break :unused;
 6192
 6193                const ty_pl = air.data(air.inst_index).ty_pl;
 6194                const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
 6195                const elem_size = ty_pl.ty.toType().childType(zcu).abiSize(zcu);
 6196
 6197                const slice_vi = try isel.use(bin_op.lhs);
 6198                var ptr_part_it = slice_vi.field(isel.air.typeOf(bin_op.lhs, ip), 0, 8);
 6199                const ptr_part_vi = try ptr_part_it.only(isel);
 6200                const ptr_part_mat = try ptr_part_vi.?.matReg(isel);
 6201                const index_vi = try isel.use(bin_op.rhs);
 6202                try isel.elemPtr(elem_ptr_ra, ptr_part_mat.ra, .add, elem_size, index_vi);
 6203                try ptr_part_mat.finish(isel);
 6204            }
 6205            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6206        },
 6207        .ptr_elem_val => {
 6208            if (isel.live_values.fetchRemove(air.inst_index)) |elem_vi| unused: {
 6209                defer elem_vi.value.deref(isel);
 6210
 6211                const bin_op = air.data(air.inst_index).bin_op;
 6212                const ptr_ty = isel.air.typeOf(bin_op.lhs, ip);
 6213                const ptr_info = ptr_ty.ptrInfo(zcu);
 6214                const elem_size = elem_vi.value.size(isel);
 6215                const elem_is_vector = elem_vi.value.isVector(isel);
 6216                if (switch (elem_size) {
 6217                    0 => unreachable,
 6218                    1, 2, 4, 8 => true,
 6219                    16 => elem_is_vector,
 6220                    else => false,
 6221                }) {
 6222                    const elem_ra = try elem_vi.value.defReg(isel) orelse break :unused;
 6223                    const base_vi = try isel.use(bin_op.lhs);
 6224                    const index_vi = try isel.use(bin_op.rhs);
 6225                    const base_mat = try base_vi.matReg(isel);
 6226                    const index_mat = try index_vi.matReg(isel);
 6227                    try isel.emit(switch (elem_size) {
 6228                        else => unreachable,
 6229                        1 => if (elem_is_vector) .ldr(elem_ra.b(), .{ .extended_register = .{
 6230                            .base = base_mat.ra.x(),
 6231                            .index = index_mat.ra.x(),
 6232                            .extend = .{ .lsl = 0 },
 6233                        } }) else switch (elem_vi.value.signedness(isel)) {
 6234                            .signed => .ldrsb(elem_ra.w(), .{ .extended_register = .{
 6235                                .base = base_mat.ra.x(),
 6236                                .index = index_mat.ra.x(),
 6237                                .extend = .{ .lsl = 0 },
 6238                            } }),
 6239                            .unsigned => .ldrb(elem_ra.w(), .{ .extended_register = .{
 6240                                .base = base_mat.ra.x(),
 6241                                .index = index_mat.ra.x(),
 6242                                .extend = .{ .lsl = 0 },
 6243                            } }),
 6244                        },
 6245                        2 => if (elem_is_vector) .ldr(elem_ra.h(), .{ .extended_register = .{
 6246                            .base = base_mat.ra.x(),
 6247                            .index = index_mat.ra.x(),
 6248                            .extend = .{ .lsl = 1 },
 6249                        } }) else switch (elem_vi.value.signedness(isel)) {
 6250                            .signed => .ldrsh(elem_ra.w(), .{ .extended_register = .{
 6251                                .base = base_mat.ra.x(),
 6252                                .index = index_mat.ra.x(),
 6253                                .extend = .{ .lsl = 1 },
 6254                            } }),
 6255                            .unsigned => .ldrh(elem_ra.w(), .{ .extended_register = .{
 6256                                .base = base_mat.ra.x(),
 6257                                .index = index_mat.ra.x(),
 6258                                .extend = .{ .lsl = 1 },
 6259                            } }),
 6260                        },
 6261                        4 => .ldr(if (elem_is_vector) elem_ra.s() else elem_ra.w(), .{ .extended_register = .{
 6262                            .base = base_mat.ra.x(),
 6263                            .index = index_mat.ra.x(),
 6264                            .extend = .{ .lsl = 2 },
 6265                        } }),
 6266                        8 => .ldr(if (elem_is_vector) elem_ra.d() else elem_ra.x(), .{ .extended_register = .{
 6267                            .base = base_mat.ra.x(),
 6268                            .index = index_mat.ra.x(),
 6269                            .extend = .{ .lsl = 3 },
 6270                        } }),
 6271                        16 => if (elem_is_vector) .ldr(elem_ra.q(), .{ .extended_register = .{
 6272                            .base = base_mat.ra.x(),
 6273                            .index = index_mat.ra.x(),
 6274                            .extend = .{ .lsl = 4 },
 6275                        } }) else unreachable,
 6276                    });
 6277                    try index_mat.finish(isel);
 6278                    try base_mat.finish(isel);
 6279                } else {
 6280                    const elem_ptr_ra = try isel.allocIntReg();
 6281                    defer isel.freeReg(elem_ptr_ra);
 6282                    if (!try elem_vi.value.load(isel, ptr_ty.elemType2(zcu), elem_ptr_ra, .{
 6283                        .@"volatile" = ptr_info.flags.is_volatile,
 6284                    })) break :unused;
 6285                    const base_vi = try isel.use(bin_op.lhs);
 6286                    const base_mat = try base_vi.matReg(isel);
 6287                    const index_vi = try isel.use(bin_op.rhs);
 6288                    try isel.elemPtr(elem_ptr_ra, base_mat.ra, .add, elem_size, index_vi);
 6289                    try base_mat.finish(isel);
 6290                }
 6291            }
 6292            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6293        },
 6294        .ptr_elem_ptr => {
 6295            if (isel.live_values.fetchRemove(air.inst_index)) |elem_ptr_vi| unused: {
 6296                defer elem_ptr_vi.value.deref(isel);
 6297                const elem_ptr_ra = try elem_ptr_vi.value.defReg(isel) orelse break :unused;
 6298
 6299                const ty_pl = air.data(air.inst_index).ty_pl;
 6300                const bin_op = isel.air.extraData(Air.Bin, ty_pl.payload).data;
 6301                const elem_size = ty_pl.ty.toType().childType(zcu).abiSize(zcu);
 6302
 6303                const base_vi = try isel.use(bin_op.lhs);
 6304                const base_mat = try base_vi.matReg(isel);
 6305                const index_vi = try isel.use(bin_op.rhs);
 6306                try isel.elemPtr(elem_ptr_ra, base_mat.ra, .add, elem_size, index_vi);
 6307                try base_mat.finish(isel);
 6308            }
 6309            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6310        },
 6311        .array_to_slice => {
 6312            if (isel.live_values.fetchRemove(air.inst_index)) |slice_vi| {
 6313                defer slice_vi.value.deref(isel);
 6314                const ty_op = air.data(air.inst_index).ty_op;
 6315                var ptr_part_it = slice_vi.value.field(ty_op.ty.toType(), 0, 8);
 6316                const ptr_part_vi = try ptr_part_it.only(isel);
 6317                try ptr_part_vi.?.move(isel, ty_op.operand);
 6318                var len_part_it = slice_vi.value.field(ty_op.ty.toType(), 8, 8);
 6319                const len_part_vi = try len_part_it.only(isel);
 6320                if (try len_part_vi.?.defReg(isel)) |len_ra| try isel.movImmediate(
 6321                    len_ra.x(),
 6322                    isel.air.typeOf(ty_op.operand, ip).childType(zcu).arrayLen(zcu),
 6323                );
 6324            }
 6325            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6326        },
 6327        .int_from_float, .int_from_float_optimized => |air_tag| {
 6328            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 6329                defer dst_vi.value.deref(isel);
 6330
 6331                const ty_op = air.data(air.inst_index).ty_op;
 6332                const dst_ty = ty_op.ty.toType();
 6333                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 6334                if (!dst_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 6335                const dst_int_info = dst_ty.intInfo(zcu);
 6336                const src_bits = src_ty.floatBits(isel.target);
 6337                switch (@max(dst_int_info.bits, src_bits)) {
 6338                    0 => unreachable,
 6339                    1...64 => {
 6340                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 6341                        const need_fcvt = switch (src_bits) {
 6342                            else => unreachable,
 6343                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 6344                            32, 64 => false,
 6345                        };
 6346                        const src_vi = try isel.use(ty_op.operand);
 6347                        const src_mat = try src_vi.matReg(isel);
 6348                        const src_ra = if (need_fcvt) try isel.allocVecReg() else src_mat.ra;
 6349                        defer if (need_fcvt) isel.freeReg(src_ra);
 6350                        const dst_reg = switch (dst_int_info.bits) {
 6351                            else => unreachable,
 6352                            1...32 => dst_ra.w(),
 6353                            33...64 => dst_ra.x(),
 6354                        };
 6355                        const src_reg = switch (src_bits) {
 6356                            else => unreachable,
 6357                            16 => if (need_fcvt) src_ra.s() else src_ra.h(),
 6358                            32 => src_ra.s(),
 6359                            64 => src_ra.d(),
 6360                        };
 6361                        try isel.emit(switch (dst_int_info.signedness) {
 6362                            .signed => .fcvtzs(dst_reg, src_reg),
 6363                            .unsigned => .fcvtzu(dst_reg, src_reg),
 6364                        });
 6365                        if (need_fcvt) try isel.emit(.fcvt(src_reg, src_mat.ra.h()));
 6366                        try src_mat.finish(isel);
 6367                    },
 6368                    65...128 => {
 6369                        try call.prepareReturn(isel);
 6370                        switch (dst_int_info.bits) {
 6371                            else => unreachable,
 6372                            1...64 => try call.returnLiveIn(isel, dst_vi.value, .r0),
 6373                            65...128 => {
 6374                                var dst_hi64_it = dst_vi.value.field(dst_ty, 8, 8);
 6375                                const dst_hi64_vi = try dst_hi64_it.only(isel);
 6376                                try call.returnLiveIn(isel, dst_hi64_vi.?, .r1);
 6377                                var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 6378                                const dst_lo64_vi = try dst_lo64_it.only(isel);
 6379                                try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
 6380                            },
 6381                        }
 6382                        try call.finishReturn(isel);
 6383
 6384                        try call.prepareCallee(isel);
 6385                        try isel.global_relocs.append(gpa, .{
 6386                            .name = switch (dst_int_info.bits) {
 6387                                else => unreachable,
 6388                                1...32 => switch (dst_int_info.signedness) {
 6389                                    .signed => switch (src_bits) {
 6390                                        else => unreachable,
 6391                                        16 => "__fixhfsi",
 6392                                        32 => "__fixsfsi",
 6393                                        64 => "__fixdfsi",
 6394                                        80 => "__fixxfsi",
 6395                                        128 => "__fixtfsi",
 6396                                    },
 6397                                    .unsigned => switch (src_bits) {
 6398                                        else => unreachable,
 6399                                        16 => "__fixunshfsi",
 6400                                        32 => "__fixunssfsi",
 6401                                        64 => "__fixunsdfsi",
 6402                                        80 => "__fixunsxfsi",
 6403                                        128 => "__fixunstfsi",
 6404                                    },
 6405                                },
 6406                                33...64 => switch (dst_int_info.signedness) {
 6407                                    .signed => switch (src_bits) {
 6408                                        else => unreachable,
 6409                                        16 => "__fixhfdi",
 6410                                        32 => "__fixsfdi",
 6411                                        64 => "__fixdfdi",
 6412                                        80 => "__fixxfdi",
 6413                                        128 => "__fixtfdi",
 6414                                    },
 6415                                    .unsigned => switch (src_bits) {
 6416                                        else => unreachable,
 6417                                        16 => "__fixunshfdi",
 6418                                        32 => "__fixunssfdi",
 6419                                        64 => "__fixunsdfdi",
 6420                                        80 => "__fixunsxfdi",
 6421                                        128 => "__fixunstfdi",
 6422                                    },
 6423                                },
 6424                                65...128 => switch (dst_int_info.signedness) {
 6425                                    .signed => switch (src_bits) {
 6426                                        else => unreachable,
 6427                                        16 => "__fixhfti",
 6428                                        32 => "__fixsfti",
 6429                                        64 => "__fixdfti",
 6430                                        80 => "__fixxfti",
 6431                                        128 => "__fixtfti",
 6432                                    },
 6433                                    .unsigned => switch (src_bits) {
 6434                                        else => unreachable,
 6435                                        16 => "__fixunshfti",
 6436                                        32 => "__fixunssfti",
 6437                                        64 => "__fixunsdfti",
 6438                                        80 => "__fixunsxfti",
 6439                                        128 => "__fixunstfti",
 6440                                    },
 6441                                },
 6442                            },
 6443                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6444                        });
 6445                        try isel.emit(.bl(0));
 6446                        try call.finishCallee(isel);
 6447
 6448                        try call.prepareParams(isel);
 6449                        const src_vi = try isel.use(ty_op.operand);
 6450                        switch (src_bits) {
 6451                            else => unreachable,
 6452                            16, 32, 64, 128 => try call.paramLiveOut(isel, src_vi, .v0),
 6453                            80 => {
 6454                                var src_hi16_it = src_vi.field(src_ty, 8, 8);
 6455                                const src_hi16_vi = try src_hi16_it.only(isel);
 6456                                try call.paramLiveOut(isel, src_hi16_vi.?, .r1);
 6457                                var src_lo64_it = src_vi.field(src_ty, 0, 8);
 6458                                const src_lo64_vi = try src_lo64_it.only(isel);
 6459                                try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
 6460                            },
 6461                        }
 6462                        try call.finishParams(isel);
 6463                    },
 6464                    else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
 6465                }
 6466            }
 6467            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6468        },
 6469        .float_from_int => |air_tag| {
 6470            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 6471                defer dst_vi.value.deref(isel);
 6472
 6473                const ty_op = air.data(air.inst_index).ty_op;
 6474                const dst_ty = ty_op.ty.toType();
 6475                const src_ty = isel.air.typeOf(ty_op.operand, ip);
 6476                const dst_bits = dst_ty.floatBits(isel.target);
 6477                if (!src_ty.isAbiInt(zcu)) return isel.fail("bad {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) });
 6478                const src_int_info = src_ty.intInfo(zcu);
 6479                switch (@max(dst_bits, src_int_info.bits)) {
 6480                    0 => unreachable,
 6481                    1...64 => {
 6482                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 6483                        const need_fcvt = switch (dst_bits) {
 6484                            else => unreachable,
 6485                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 6486                            32, 64 => false,
 6487                        };
 6488                        if (need_fcvt) try isel.emit(.fcvt(dst_ra.h(), dst_ra.s()));
 6489                        const src_vi = try isel.use(ty_op.operand);
 6490                        const src_mat = try src_vi.matReg(isel);
 6491                        const dst_reg = switch (dst_bits) {
 6492                            else => unreachable,
 6493                            16 => if (need_fcvt) dst_ra.s() else dst_ra.h(),
 6494                            32 => dst_ra.s(),
 6495                            64 => dst_ra.d(),
 6496                        };
 6497                        const src_reg = switch (src_int_info.bits) {
 6498                            else => unreachable,
 6499                            1...32 => src_mat.ra.w(),
 6500                            33...64 => src_mat.ra.x(),
 6501                        };
 6502                        try isel.emit(switch (src_int_info.signedness) {
 6503                            .signed => .scvtf(dst_reg, src_reg),
 6504                            .unsigned => .ucvtf(dst_reg, src_reg),
 6505                        });
 6506                        try src_mat.finish(isel);
 6507                    },
 6508                    65...128 => {
 6509                        try call.prepareReturn(isel);
 6510                        switch (dst_bits) {
 6511                            else => unreachable,
 6512                            16, 32, 64, 128 => try call.returnLiveIn(isel, dst_vi.value, .v0),
 6513                            80 => {
 6514                                var dst_hi16_it = dst_vi.value.field(dst_ty, 8, 8);
 6515                                const dst_hi16_vi = try dst_hi16_it.only(isel);
 6516                                try call.returnLiveIn(isel, dst_hi16_vi.?, .r1);
 6517                                var dst_lo64_it = dst_vi.value.field(dst_ty, 0, 8);
 6518                                const dst_lo64_vi = try dst_lo64_it.only(isel);
 6519                                try call.returnLiveIn(isel, dst_lo64_vi.?, .r0);
 6520                            },
 6521                        }
 6522                        try call.finishReturn(isel);
 6523
 6524                        try call.prepareCallee(isel);
 6525                        try isel.global_relocs.append(gpa, .{
 6526                            .name = switch (src_int_info.bits) {
 6527                                else => unreachable,
 6528                                1...32 => switch (src_int_info.signedness) {
 6529                                    .signed => switch (dst_bits) {
 6530                                        else => unreachable,
 6531                                        16 => "__floatsihf",
 6532                                        32 => "__floatsisf",
 6533                                        64 => "__floatsidf",
 6534                                        80 => "__floatsixf",
 6535                                        128 => "__floatsitf",
 6536                                    },
 6537                                    .unsigned => switch (dst_bits) {
 6538                                        else => unreachable,
 6539                                        16 => "__floatunsihf",
 6540                                        32 => "__floatunsisf",
 6541                                        64 => "__floatunsidf",
 6542                                        80 => "__floatunsixf",
 6543                                        128 => "__floatunsitf",
 6544                                    },
 6545                                },
 6546                                33...64 => switch (src_int_info.signedness) {
 6547                                    .signed => switch (dst_bits) {
 6548                                        else => unreachable,
 6549                                        16 => "__floatdihf",
 6550                                        32 => "__floatdisf",
 6551                                        64 => "__floatdidf",
 6552                                        80 => "__floatdixf",
 6553                                        128 => "__floatditf",
 6554                                    },
 6555                                    .unsigned => switch (dst_bits) {
 6556                                        else => unreachable,
 6557                                        16 => "__floatundihf",
 6558                                        32 => "__floatundisf",
 6559                                        64 => "__floatundidf",
 6560                                        80 => "__floatundixf",
 6561                                        128 => "__floatunditf",
 6562                                    },
 6563                                },
 6564                                65...128 => switch (src_int_info.signedness) {
 6565                                    .signed => switch (dst_bits) {
 6566                                        else => unreachable,
 6567                                        16 => "__floattihf",
 6568                                        32 => "__floattisf",
 6569                                        64 => "__floattidf",
 6570                                        80 => "__floattixf",
 6571                                        128 => "__floattitf",
 6572                                    },
 6573                                    .unsigned => switch (dst_bits) {
 6574                                        else => unreachable,
 6575                                        16 => "__floatuntihf",
 6576                                        32 => "__floatuntisf",
 6577                                        64 => "__floatuntidf",
 6578                                        80 => "__floatuntixf",
 6579                                        128 => "__floatuntitf",
 6580                                    },
 6581                                },
 6582                            },
 6583                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6584                        });
 6585                        try isel.emit(.bl(0));
 6586                        try call.finishCallee(isel);
 6587
 6588                        try call.prepareParams(isel);
 6589                        const src_vi = try isel.use(ty_op.operand);
 6590                        switch (src_int_info.bits) {
 6591                            else => unreachable,
 6592                            1...64 => try call.paramLiveOut(isel, src_vi, .r0),
 6593                            65...128 => {
 6594                                var src_hi64_it = src_vi.field(src_ty, 8, 8);
 6595                                const src_hi64_vi = try src_hi64_it.only(isel);
 6596                                try call.paramLiveOut(isel, src_hi64_vi.?, .r1);
 6597                                var src_lo64_it = src_vi.field(src_ty, 0, 8);
 6598                                const src_lo64_vi = try src_lo64_it.only(isel);
 6599                                try call.paramLiveOut(isel, src_lo64_vi.?, .r0);
 6600                            },
 6601                        }
 6602                        try call.finishParams(isel);
 6603                    },
 6604                    else => return isel.fail("too big {t} {f} {f}", .{ air_tag, isel.fmtType(dst_ty), isel.fmtType(src_ty) }),
 6605                }
 6606            }
 6607            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6608        },
 6609        .memset, .memset_safe => |air_tag| {
 6610            const bin_op = air.data(air.inst_index).bin_op;
 6611            const dst_ty = isel.air.typeOf(bin_op.lhs, ip);
 6612            const dst_info = dst_ty.ptrInfo(zcu);
 6613            const fill_byte: union(enum) { constant: u8, value: Air.Inst.Ref } = fill_byte: {
 6614                if (bin_op.rhs.toInterned()) |fill_val| {
 6615                    if (ip.isUndef(fill_val)) switch (air_tag) {
 6616                        else => unreachable,
 6617                        .memset => break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
 6618                        .memset_safe => break :fill_byte .{ .constant = 0xaa },
 6619                    };
 6620                    if (try isel.hasRepeatedByteRepr(.fromInterned(fill_val))) |fill_byte|
 6621                        break :fill_byte .{ .constant = fill_byte };
 6622                }
 6623                switch (dst_ty.elemType2(zcu).abiSize(zcu)) {
 6624                    0 => unreachable,
 6625                    1 => break :fill_byte .{ .value = bin_op.rhs },
 6626                    2, 4, 8 => |size| {
 6627                        const dst_vi = try isel.use(bin_op.lhs);
 6628                        const ptr_ra = try isel.allocIntReg();
 6629                        const fill_vi = try isel.use(bin_op.rhs);
 6630                        const fill_mat = try fill_vi.matReg(isel);
 6631                        const len_mat: Value.Materialize = len_mat: switch (dst_info.flags.size) {
 6632                            .one => .{ .vi = undefined, .ra = try isel.allocIntReg() },
 6633                            .many => unreachable,
 6634                            .slice => {
 6635                                var dst_len_it = dst_vi.field(dst_ty, 8, 8);
 6636                                const dst_len_vi = try dst_len_it.only(isel);
 6637                                break :len_mat try dst_len_vi.?.matReg(isel);
 6638                            },
 6639                            .c => unreachable,
 6640                        };
 6641
 6642                        const skip_label = isel.instructions.items.len;
 6643                        _ = try isel.instructions.addOne(gpa);
 6644                        try isel.emit(.sub(len_mat.ra.x(), len_mat.ra.x(), .{ .immediate = 1 }));
 6645                        try isel.emit(switch (size) {
 6646                            else => unreachable,
 6647                            2 => .strh(fill_mat.ra.w(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 2 } }),
 6648                            4 => .str(fill_mat.ra.w(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 4 } }),
 6649                            8 => .str(fill_mat.ra.x(), .{ .post_index = .{ .base = ptr_ra.x(), .index = 8 } }),
 6650                        });
 6651                        isel.instructions.items[skip_label] = .cbnz(
 6652                            len_mat.ra.x(),
 6653                            -@as(i21, @intCast((isel.instructions.items.len - 1 - skip_label) << 2)),
 6654                        );
 6655                        switch (dst_info.flags.size) {
 6656                            .one => {
 6657                                const len_imm = ZigType.fromInterned(dst_info.child).arrayLen(zcu);
 6658                                assert(len_imm > 0);
 6659                                try isel.movImmediate(len_mat.ra.x(), len_imm);
 6660                                isel.freeReg(len_mat.ra);
 6661                                try fill_mat.finish(isel);
 6662                                isel.freeReg(ptr_ra);
 6663                                try dst_vi.liveOut(isel, ptr_ra);
 6664                            },
 6665                            .many => unreachable,
 6666                            .slice => {
 6667                                try isel.emit(.cbz(
 6668                                    len_mat.ra.x(),
 6669                                    @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 6670                                ));
 6671                                try len_mat.finish(isel);
 6672                                try fill_mat.finish(isel);
 6673                                isel.freeReg(ptr_ra);
 6674                                var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
 6675                                const dst_ptr_vi = try dst_ptr_it.only(isel);
 6676                                try dst_ptr_vi.?.liveOut(isel, ptr_ra);
 6677                            },
 6678                            .c => unreachable,
 6679                        }
 6680
 6681                        break :air_tag if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6682                    },
 6683                    else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(dst_ty) }),
 6684                }
 6685            };
 6686
 6687            try call.prepareReturn(isel);
 6688            try call.finishReturn(isel);
 6689
 6690            try call.prepareCallee(isel);
 6691            try isel.global_relocs.append(gpa, .{
 6692                .name = "memset",
 6693                .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6694            });
 6695            try isel.emit(.bl(0));
 6696            try call.finishCallee(isel);
 6697
 6698            try call.prepareParams(isel);
 6699            const dst_vi = try isel.use(bin_op.lhs);
 6700            switch (dst_info.flags.size) {
 6701                .one => {
 6702                    try isel.movImmediate(.x2, ZigType.fromInterned(dst_info.child).abiSize(zcu));
 6703                    switch (fill_byte) {
 6704                        .constant => |byte| try isel.movImmediate(.w1, byte),
 6705                        .value => |byte| try call.paramLiveOut(isel, try isel.use(byte), .r1),
 6706                    }
 6707                    try call.paramLiveOut(isel, dst_vi, .r0);
 6708                },
 6709                .many => unreachable,
 6710                .slice => {
 6711                    var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
 6712                    const dst_ptr_vi = try dst_ptr_it.only(isel);
 6713                    var dst_len_it = dst_vi.field(dst_ty, 8, 8);
 6714                    const dst_len_vi = try dst_len_it.only(isel);
 6715                    try isel.elemPtr(.r2, .zr, .add, ZigType.fromInterned(dst_info.child).abiSize(zcu), dst_len_vi.?);
 6716                    switch (fill_byte) {
 6717                        .constant => |byte| try isel.movImmediate(.w1, byte),
 6718                        .value => |byte| try call.paramLiveOut(isel, try isel.use(byte), .r1),
 6719                    }
 6720                    try call.paramLiveOut(isel, dst_ptr_vi.?, .r0);
 6721                },
 6722                .c => unreachable,
 6723            }
 6724            try call.finishParams(isel);
 6725
 6726            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6727        },
 6728        .memcpy, .memmove => |air_tag| {
 6729            const bin_op = air.data(air.inst_index).bin_op;
 6730            const dst_ty = isel.air.typeOf(bin_op.lhs, ip);
 6731            const dst_info = dst_ty.ptrInfo(zcu);
 6732
 6733            try call.prepareReturn(isel);
 6734            try call.finishReturn(isel);
 6735
 6736            try call.prepareCallee(isel);
 6737            try isel.global_relocs.append(gpa, .{
 6738                .name = @tagName(air_tag),
 6739                .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6740            });
 6741            try isel.emit(.bl(0));
 6742            try call.finishCallee(isel);
 6743
 6744            try call.prepareParams(isel);
 6745            switch (dst_info.flags.size) {
 6746                .one => {
 6747                    const dst_vi = try isel.use(bin_op.lhs);
 6748                    const src_vi = try isel.use(bin_op.rhs);
 6749                    try isel.movImmediate(.x2, ZigType.fromInterned(dst_info.child).abiSize(zcu));
 6750                    try call.paramLiveOut(isel, src_vi, .r1);
 6751                    try call.paramLiveOut(isel, dst_vi, .r0);
 6752                },
 6753                .many => unreachable,
 6754                .slice => {
 6755                    const dst_vi = try isel.use(bin_op.lhs);
 6756                    var dst_ptr_it = dst_vi.field(dst_ty, 0, 8);
 6757                    const dst_ptr_vi = try dst_ptr_it.only(isel);
 6758                    var dst_len_it = dst_vi.field(dst_ty, 8, 8);
 6759                    const dst_len_vi = try dst_len_it.only(isel);
 6760                    const src_vi = try isel.use(bin_op.rhs);
 6761                    try isel.elemPtr(.r2, .zr, .add, ZigType.fromInterned(dst_info.child).abiSize(zcu), dst_len_vi.?);
 6762                    try call.paramLiveOut(isel, src_vi, .r1);
 6763                    try call.paramLiveOut(isel, dst_ptr_vi.?, .r0);
 6764                },
 6765                .c => unreachable,
 6766            }
 6767            try call.finishParams(isel);
 6768
 6769            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6770        },
 6771        .atomic_load => {
 6772            const atomic_load = air.data(air.inst_index).atomic_load;
 6773            const ptr_ty = isel.air.typeOf(atomic_load.ptr, ip);
 6774            const ptr_info = ptr_ty.ptrInfo(zcu);
 6775            if (atomic_load.order != .unordered) return isel.fail("ordered atomic load", .{});
 6776            if (ptr_info.packed_offset.host_size > 0) return isel.fail("packed atomic load", .{});
 6777
 6778            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| {
 6779                defer dst_vi.value.deref(isel);
 6780                var ptr_mat: ?Value.Materialize = null;
 6781                var dst_part_it = dst_vi.value.parts(isel);
 6782                while (dst_part_it.next()) |dst_part_vi| {
 6783                    const dst_ra = try dst_part_vi.defReg(isel) orelse continue;
 6784                    if (ptr_mat == null) {
 6785                        const ptr_vi = try isel.use(atomic_load.ptr);
 6786                        ptr_mat = try ptr_vi.matReg(isel);
 6787                    }
 6788                    try isel.emit(switch (dst_part_vi.size(isel)) {
 6789                        else => |size| return isel.fail("bad atomic load size of {d} from {f}", .{
 6790                            size, isel.fmtType(ptr_ty),
 6791                        }),
 6792                        1 => switch (dst_part_vi.signedness(isel)) {
 6793                            .signed => .ldrsb(dst_ra.w(), .{ .unsigned_offset = .{
 6794                                .base = ptr_mat.?.ra.x(),
 6795                                .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6796                            } }),
 6797                            .unsigned => .ldrb(dst_ra.w(), .{ .unsigned_offset = .{
 6798                                .base = ptr_mat.?.ra.x(),
 6799                                .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6800                            } }),
 6801                        },
 6802                        2 => switch (dst_part_vi.signedness(isel)) {
 6803                            .signed => .ldrsh(dst_ra.w(), .{ .unsigned_offset = .{
 6804                                .base = ptr_mat.?.ra.x(),
 6805                                .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6806                            } }),
 6807                            .unsigned => .ldrh(dst_ra.w(), .{ .unsigned_offset = .{
 6808                                .base = ptr_mat.?.ra.x(),
 6809                                .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6810                            } }),
 6811                        },
 6812                        4 => .ldr(dst_ra.w(), .{ .unsigned_offset = .{
 6813                            .base = ptr_mat.?.ra.x(),
 6814                            .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6815                        } }),
 6816                        8 => .ldr(dst_ra.x(), .{ .unsigned_offset = .{
 6817                            .base = ptr_mat.?.ra.x(),
 6818                            .offset = @intCast(dst_part_vi.get(isel).offset_from_parent),
 6819                        } }),
 6820                    });
 6821                }
 6822                if (ptr_mat) |mat| try mat.finish(isel);
 6823            } else if (ptr_info.flags.is_volatile) return isel.fail("volatile atomic load", .{});
 6824
 6825            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6826        },
 6827        .error_name => {
 6828            if (isel.live_values.fetchRemove(air.inst_index)) |name_vi| unused: {
 6829                defer name_vi.value.deref(isel);
 6830                var ptr_part_it = name_vi.value.field(.slice_const_u8_sentinel_0, 0, 8);
 6831                const ptr_part_vi = try ptr_part_it.only(isel);
 6832                const ptr_part_ra = try ptr_part_vi.?.defReg(isel);
 6833                var len_part_it = name_vi.value.field(.slice_const_u8_sentinel_0, 8, 8);
 6834                const len_part_vi = try len_part_it.only(isel);
 6835                const len_part_ra = try len_part_vi.?.defReg(isel);
 6836                if (ptr_part_ra == null and len_part_ra == null) break :unused;
 6837
 6838                const un_op = air.data(air.inst_index).un_op;
 6839                const error_vi = try isel.use(un_op);
 6840                const error_mat = try error_vi.matReg(isel);
 6841                const ptr_ra = try isel.allocIntReg();
 6842                defer isel.freeReg(ptr_ra);
 6843                const start_ra, const end_ra = range_ras: {
 6844                    const name_lock: RegLock = if (len_part_ra != null) if (ptr_part_ra) |name_ptr_ra|
 6845                        isel.tryLockReg(name_ptr_ra)
 6846                    else
 6847                        .empty else .empty;
 6848                    defer name_lock.unlock(isel);
 6849                    break :range_ras .{ try isel.allocIntReg(), try isel.allocIntReg() };
 6850                };
 6851                defer {
 6852                    isel.freeReg(start_ra);
 6853                    isel.freeReg(end_ra);
 6854                }
 6855                if (len_part_ra) |name_len_ra| try isel.emit(.sub(
 6856                    name_len_ra.w(),
 6857                    end_ra.w(),
 6858                    .{ .register = start_ra.w() },
 6859                ));
 6860                if (ptr_part_ra) |name_ptr_ra| try isel.emit(.add(
 6861                    name_ptr_ra.x(),
 6862                    ptr_ra.x(),
 6863                    .{ .extended_register = .{
 6864                        .register = start_ra.w(),
 6865                        .extend = .{ .uxtw = 0 },
 6866                    } },
 6867                ));
 6868                if (len_part_ra) |_| try isel.emit(.sub(end_ra.w(), end_ra.w(), .{ .immediate = 1 }));
 6869                try isel.emit(.ldp(start_ra.w(), end_ra.w(), .{ .base = start_ra.x() }));
 6870                try isel.emit(.add(start_ra.x(), ptr_ra.x(), .{ .extended_register = .{
 6871                    .register = error_mat.ra.w(),
 6872                    .extend = switch (zcu.errorSetBits()) {
 6873                        else => unreachable,
 6874                        1...8 => .{ .uxtb = 2 },
 6875                        9...16 => .{ .uxth = 2 },
 6876                        17...32 => .{ .uxtw = 2 },
 6877                    },
 6878                } }));
 6879                try isel.lazy_relocs.append(gpa, .{
 6880                    .symbol = .{ .kind = .const_data, .ty = .anyerror_type },
 6881                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6882                });
 6883                try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
 6884                try isel.lazy_relocs.append(gpa, .{
 6885                    .symbol = .{ .kind = .const_data, .ty = .anyerror_type },
 6886                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 6887                });
 6888                try isel.emit(.adrp(ptr_ra.x(), 0));
 6889                try error_mat.finish(isel);
 6890            }
 6891            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6892        },
 6893        .aggregate_init => {
 6894            if (isel.live_values.fetchRemove(air.inst_index)) |agg_vi| {
 6895                defer agg_vi.value.deref(isel);
 6896
 6897                const ty_pl = air.data(air.inst_index).ty_pl;
 6898                const agg_ty = ty_pl.ty.toType();
 6899                switch (ip.indexToKey(agg_ty.toIntern())) {
 6900                    .array_type => |array_type| {
 6901                        const elems: []const Air.Inst.Ref =
 6902                            @ptrCast(isel.air.extra.items[ty_pl.payload..][0..@intCast(array_type.len)]);
 6903                        var elem_offset: u64 = 0;
 6904                        const elem_size = ZigType.fromInterned(array_type.child).abiSize(zcu);
 6905                        for (elems) |elem| {
 6906                            var agg_part_it = agg_vi.value.field(agg_ty, elem_offset, elem_size);
 6907                            const agg_part_vi = try agg_part_it.only(isel);
 6908                            try agg_part_vi.?.move(isel, elem);
 6909                            elem_offset += elem_size;
 6910                        }
 6911                        switch (array_type.sentinel) {
 6912                            .none => {},
 6913                            else => |sentinel| {
 6914                                var agg_part_it = agg_vi.value.field(agg_ty, elem_offset, elem_size);
 6915                                const agg_part_vi = try agg_part_it.only(isel);
 6916                                try agg_part_vi.?.move(isel, .fromIntern(sentinel));
 6917                            },
 6918                        }
 6919                    },
 6920                    .struct_type => {
 6921                        const loaded_struct = ip.loadStructType(agg_ty.toIntern());
 6922                        const elems: []const Air.Inst.Ref =
 6923                            @ptrCast(isel.air.extra.items[ty_pl.payload..][0..loaded_struct.field_types.len]);
 6924                        var field_offset: u64 = 0;
 6925                        var field_it = loaded_struct.iterateRuntimeOrder(ip);
 6926                        while (field_it.next()) |field_index| {
 6927                            const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
 6928                            field_offset = field_ty.structFieldAlignment(
 6929                                loaded_struct.fieldAlign(ip, field_index),
 6930                                loaded_struct.layout,
 6931                                zcu,
 6932                            ).forward(field_offset);
 6933                            const field_size = field_ty.abiSize(zcu);
 6934                            if (field_size == 0) continue;
 6935                            var agg_part_it = agg_vi.value.field(agg_ty, field_offset, field_size);
 6936                            const agg_part_vi = try agg_part_it.only(isel);
 6937                            try agg_part_vi.?.move(isel, elems[field_index]);
 6938                            field_offset += field_size;
 6939                        }
 6940                        assert(loaded_struct.flagsUnordered(ip).alignment.forward(field_offset) == agg_vi.value.size(isel));
 6941                    },
 6942                    .tuple_type => |tuple_type| {
 6943                        const elems: []const Air.Inst.Ref =
 6944                            @ptrCast(isel.air.extra.items[ty_pl.payload..][0..tuple_type.types.len]);
 6945                        var tuple_align: InternPool.Alignment = .@"1";
 6946                        var field_offset: u64 = 0;
 6947                        for (
 6948                            tuple_type.types.get(ip),
 6949                            tuple_type.values.get(ip),
 6950                            elems,
 6951                        ) |field_ty_index, field_val, elem| {
 6952                            if (field_val != .none) continue;
 6953                            const field_ty: ZigType = .fromInterned(field_ty_index);
 6954                            const field_align = field_ty.abiAlignment(zcu);
 6955                            tuple_align = tuple_align.maxStrict(field_align);
 6956                            field_offset = field_align.forward(field_offset);
 6957                            const field_size = field_ty.abiSize(zcu);
 6958                            if (field_size == 0) continue;
 6959                            var agg_part_it = agg_vi.value.field(agg_ty, field_offset, field_size);
 6960                            const agg_part_vi = try agg_part_it.only(isel);
 6961                            try agg_part_vi.?.move(isel, elem);
 6962                            field_offset += field_size;
 6963                        }
 6964                        assert(tuple_align.forward(field_offset) == agg_vi.value.size(isel));
 6965                    },
 6966                    else => return isel.fail("aggregate init {f}", .{isel.fmtType(agg_ty)}),
 6967                }
 6968            }
 6969            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 6970        },
 6971        .union_init => |air_tag| {
 6972            if (isel.live_values.fetchRemove(air.inst_index)) |union_vi| unused: {
 6973                defer union_vi.value.deref(isel);
 6974
 6975                const ty_pl = air.data(air.inst_index).ty_pl;
 6976                const extra = isel.air.extraData(Air.UnionInit, ty_pl.payload).data;
 6977                const union_ty = ty_pl.ty.toType();
 6978                const loaded_union = ip.loadUnionType(union_ty.toIntern());
 6979                const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
 6980
 6981                if (union_layout.tag_size > 0) unused_tag: {
 6982                    const loaded_tag = loaded_union.loadTagType(ip);
 6983                    var tag_it = union_vi.value.field(union_ty, union_layout.tagOffset(), union_layout.tag_size);
 6984                    const tag_vi = try tag_it.only(isel);
 6985                    const tag_ra = try tag_vi.?.defReg(isel) orelse break :unused_tag;
 6986                    switch (union_layout.tag_size) {
 6987                        0 => unreachable,
 6988                        1...4 => try isel.movImmediate(tag_ra.w(), @as(u32, switch (loaded_tag.values.len) {
 6989                            0 => extra.field_index,
 6990                            else => switch (ip.indexToKey(loaded_tag.values.get(ip)[extra.field_index]).int.storage) {
 6991                                .u64 => |imm| @intCast(imm),
 6992                                .i64 => |imm| @bitCast(@as(i32, @intCast(imm))),
 6993                                else => unreachable,
 6994                            },
 6995                        })),
 6996                        5...8 => try isel.movImmediate(tag_ra.x(), switch (loaded_tag.values.len) {
 6997                            0 => extra.field_index,
 6998                            else => switch (ip.indexToKey(loaded_tag.values.get(ip)[extra.field_index]).int.storage) {
 6999                                .u64 => |imm| imm,
 7000                                .i64 => |imm| @bitCast(imm),
 7001                                else => unreachable,
 7002                            },
 7003                        }),
 7004                        else => return isel.fail("too big {t} {f}", .{ air_tag, isel.fmtType(union_ty) }),
 7005                    }
 7006                }
 7007                var payload_it = union_vi.value.field(union_ty, union_layout.payloadOffset(), union_layout.payload_size);
 7008                const payload_vi = try payload_it.only(isel);
 7009                try payload_vi.?.defAddr(isel, union_ty, .{ .root_vi = union_vi.value }) orelse break :unused;
 7010
 7011                try call.prepareReturn(isel);
 7012                try call.finishReturn(isel);
 7013
 7014                try call.prepareCallee(isel);
 7015                try isel.global_relocs.append(gpa, .{
 7016                    .name = "memcpy",
 7017                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7018                });
 7019                try isel.emit(.bl(0));
 7020                try call.finishCallee(isel);
 7021
 7022                try call.prepareParams(isel);
 7023                const init_vi = try isel.use(extra.init);
 7024                try isel.movImmediate(.x2, init_vi.size(isel));
 7025                try call.paramAddress(isel, init_vi, .r1);
 7026                try call.paramAddress(isel, payload_vi.?, .r0);
 7027                try call.finishParams(isel);
 7028            }
 7029            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7030        },
 7031        .prefetch => {
 7032            const prefetch = air.data(air.inst_index).prefetch;
 7033            if (!(prefetch.rw == .write and prefetch.cache == .instruction)) {
 7034                const maybe_slice_ty = isel.air.typeOf(prefetch.ptr, ip);
 7035                const maybe_slice_vi = try isel.use(prefetch.ptr);
 7036                const ptr_vi = if (maybe_slice_ty.isSlice(zcu)) ptr_vi: {
 7037                    var ptr_part_it = maybe_slice_vi.field(maybe_slice_ty, 0, 8);
 7038                    const ptr_part_vi = try ptr_part_it.only(isel);
 7039                    break :ptr_vi ptr_part_vi.?;
 7040                } else maybe_slice_vi;
 7041                const ptr_mat = try ptr_vi.matReg(isel);
 7042                try isel.emit(.prfm(.{
 7043                    .policy = switch (prefetch.locality) {
 7044                        1, 2, 3 => .keep,
 7045                        0 => .strm,
 7046                    },
 7047                    .target = switch (prefetch.locality) {
 7048                        0, 3 => .l1,
 7049                        2 => .l2,
 7050                        1 => .l3,
 7051                    },
 7052                    .type = switch (prefetch.rw) {
 7053                        .read => switch (prefetch.cache) {
 7054                            .data => .pld,
 7055                            .instruction => .pli,
 7056                        },
 7057                        .write => switch (prefetch.cache) {
 7058                            .data => .pst,
 7059                            .instruction => unreachable,
 7060                        },
 7061                    },
 7062                }, .{ .base = ptr_mat.ra.x() }));
 7063                try ptr_mat.finish(isel);
 7064            }
 7065            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7066        },
 7067        .mul_add => {
 7068            if (isel.live_values.fetchRemove(air.inst_index)) |res_vi| unused: {
 7069                defer res_vi.value.deref(isel);
 7070
 7071                const pl_op = air.data(air.inst_index).pl_op;
 7072                const bin_op = isel.air.extraData(Air.Bin, pl_op.payload).data;
 7073                const ty = isel.air.typeOf(pl_op.operand, ip);
 7074                switch (ty.floatBits(isel.target)) {
 7075                    else => unreachable,
 7076                    16, 32, 64 => |bits| {
 7077                        const res_ra = try res_vi.value.defReg(isel) orelse break :unused;
 7078                        const need_fcvt = switch (bits) {
 7079                            else => unreachable,
 7080                            16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 7081                            32, 64 => false,
 7082                        };
 7083                        if (need_fcvt) try isel.emit(.fcvt(res_ra.h(), res_ra.s()));
 7084                        const lhs_vi = try isel.use(bin_op.lhs);
 7085                        const rhs_vi = try isel.use(bin_op.rhs);
 7086                        const addend_vi = try isel.use(pl_op.operand);
 7087                        const lhs_mat = try lhs_vi.matReg(isel);
 7088                        const rhs_mat = try rhs_vi.matReg(isel);
 7089                        const addend_mat = try addend_vi.matReg(isel);
 7090                        const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 7091                        defer if (need_fcvt) isel.freeReg(lhs_ra);
 7092                        const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 7093                        defer if (need_fcvt) isel.freeReg(rhs_ra);
 7094                        const addend_ra = if (need_fcvt) try isel.allocVecReg() else addend_mat.ra;
 7095                        defer if (need_fcvt) isel.freeReg(addend_ra);
 7096                        try isel.emit(bits: switch (bits) {
 7097                            else => unreachable,
 7098                            16 => if (need_fcvt)
 7099                                continue :bits 32
 7100                            else
 7101                                .fmadd(res_ra.h(), lhs_ra.h(), rhs_ra.h(), addend_ra.h()),
 7102                            32 => .fmadd(res_ra.s(), lhs_ra.s(), rhs_ra.s(), addend_ra.s()),
 7103                            64 => .fmadd(res_ra.d(), lhs_ra.d(), rhs_ra.d(), addend_ra.d()),
 7104                        });
 7105                        if (need_fcvt) {
 7106                            try isel.emit(.fcvt(addend_ra.s(), addend_mat.ra.h()));
 7107                            try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 7108                            try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 7109                        }
 7110                        try addend_mat.finish(isel);
 7111                        try rhs_mat.finish(isel);
 7112                        try lhs_mat.finish(isel);
 7113                    },
 7114                    80, 128 => |bits| {
 7115                        try call.prepareReturn(isel);
 7116                        switch (bits) {
 7117                            else => unreachable,
 7118                            16, 32, 64, 128 => try call.returnLiveIn(isel, res_vi.value, .v0),
 7119                            80 => {
 7120                                var res_hi16_it = res_vi.value.field(ty, 8, 8);
 7121                                const res_hi16_vi = try res_hi16_it.only(isel);
 7122                                try call.returnLiveIn(isel, res_hi16_vi.?, .r1);
 7123                                var res_lo64_it = res_vi.value.field(ty, 0, 8);
 7124                                const res_lo64_vi = try res_lo64_it.only(isel);
 7125                                try call.returnLiveIn(isel, res_lo64_vi.?, .r0);
 7126                            },
 7127                        }
 7128                        try call.finishReturn(isel);
 7129
 7130                        try call.prepareCallee(isel);
 7131                        try isel.global_relocs.append(gpa, .{
 7132                            .name = switch (bits) {
 7133                                else => unreachable,
 7134                                16 => "__fmah",
 7135                                32 => "fmaf",
 7136                                64 => "fma",
 7137                                80 => "__fmax",
 7138                                128 => "fmaq",
 7139                            },
 7140                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7141                        });
 7142                        try isel.emit(.bl(0));
 7143                        try call.finishCallee(isel);
 7144
 7145                        try call.prepareParams(isel);
 7146                        const lhs_vi = try isel.use(bin_op.lhs);
 7147                        const rhs_vi = try isel.use(bin_op.rhs);
 7148                        const addend_vi = try isel.use(pl_op.operand);
 7149                        switch (bits) {
 7150                            else => unreachable,
 7151                            16, 32, 64, 128 => {
 7152                                try call.paramLiveOut(isel, addend_vi, .v2);
 7153                                try call.paramLiveOut(isel, rhs_vi, .v1);
 7154                                try call.paramLiveOut(isel, lhs_vi, .v0);
 7155                            },
 7156                            80 => {
 7157                                var addend_hi16_it = addend_vi.field(ty, 8, 8);
 7158                                const addend_hi16_vi = try addend_hi16_it.only(isel);
 7159                                try call.paramLiveOut(isel, addend_hi16_vi.?, .r5);
 7160                                var addend_lo64_it = addend_vi.field(ty, 0, 8);
 7161                                const addend_lo64_vi = try addend_lo64_it.only(isel);
 7162                                try call.paramLiveOut(isel, addend_lo64_vi.?, .r4);
 7163                                var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 7164                                const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 7165                                try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 7166                                var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 7167                                const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 7168                                try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 7169                                var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 7170                                const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 7171                                try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 7172                                var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 7173                                const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 7174                                try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 7175                            },
 7176                        }
 7177                        try call.finishParams(isel);
 7178                    },
 7179                }
 7180            }
 7181            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7182        },
 7183        .field_parent_ptr => {
 7184            if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
 7185                defer dst_vi.value.deref(isel);
 7186                const ty_pl = air.data(air.inst_index).ty_pl;
 7187                const extra = isel.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
 7188                switch (codegen.fieldOffset(
 7189                    ty_pl.ty.toType(),
 7190                    isel.air.typeOf(extra.field_ptr, ip),
 7191                    extra.field_index,
 7192                    zcu,
 7193                )) {
 7194                    0 => try dst_vi.value.move(isel, extra.field_ptr),
 7195                    else => |field_offset| {
 7196                        const dst_ra = try dst_vi.value.defReg(isel) orelse break :unused;
 7197                        const src_vi = try isel.use(extra.field_ptr);
 7198                        const src_mat = try src_vi.matReg(isel);
 7199                        const lo12: u12 = @truncate(field_offset >> 0);
 7200                        const hi12: u12 = @intCast(field_offset >> 12);
 7201                        if (hi12 > 0) try isel.emit(.sub(
 7202                            dst_ra.x(),
 7203                            if (lo12 > 0) dst_ra.x() else src_mat.ra.x(),
 7204                            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 7205                        ));
 7206                        if (lo12 > 0) try isel.emit(.sub(dst_ra.x(), src_mat.ra.x(), .{ .immediate = lo12 }));
 7207                        try src_mat.finish(isel);
 7208                    },
 7209                }
 7210            }
 7211            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7212        },
 7213        .wasm_memory_size, .wasm_memory_grow => unreachable,
 7214        .cmp_lt_errors_len => {
 7215            if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: {
 7216                defer is_vi.value.deref(isel);
 7217                const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
 7218                try isel.emit(.csinc(is_ra.w(), .wzr, .wzr, .invert(.ls)));
 7219
 7220                const un_op = air.data(air.inst_index).un_op;
 7221                const error_vi = try isel.use(un_op);
 7222                const error_mat = try error_vi.matReg(isel);
 7223                const ptr_ra = try isel.allocIntReg();
 7224                defer isel.freeReg(ptr_ra);
 7225                try isel.emit(.subs(.wzr, error_mat.ra.w(), .{ .register = ptr_ra.w() }));
 7226                try isel.lazy_relocs.append(gpa, .{
 7227                    .symbol = .{ .kind = .const_data, .ty = .anyerror_type },
 7228                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7229                });
 7230                try isel.emit(.ldr(ptr_ra.w(), .{ .base = ptr_ra.x() }));
 7231                try isel.lazy_relocs.append(gpa, .{
 7232                    .symbol = .{ .kind = .const_data, .ty = .anyerror_type },
 7233                    .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7234                });
 7235                try isel.emit(.adrp(ptr_ra.x(), 0));
 7236                try error_mat.finish(isel);
 7237            }
 7238            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7239        },
 7240        .runtime_nav_ptr => {
 7241            if (isel.live_values.fetchRemove(air.inst_index)) |ptr_vi| unused: {
 7242                defer ptr_vi.value.deref(isel);
 7243                const ptr_ra = try ptr_vi.value.defReg(isel) orelse break :unused;
 7244
 7245                const ty_nav = air.data(air.inst_index).ty_nav;
 7246                if (ZigType.fromInterned(ip.getNav(ty_nav.nav).typeOf(ip)).isFnOrHasRuntimeBits(zcu)) switch (true) {
 7247                    false => {
 7248                        try isel.nav_relocs.append(gpa, .{
 7249                            .nav = ty_nav.nav,
 7250                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7251                        });
 7252                        try isel.emit(.adr(ptr_ra.x(), 0));
 7253                    },
 7254                    true => {
 7255                        try isel.nav_relocs.append(gpa, .{
 7256                            .nav = ty_nav.nav,
 7257                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7258                        });
 7259                        if (ip.getNav(ty_nav.nav).getExtern(ip)) |_|
 7260                            try isel.emit(.ldr(ptr_ra.x(), .{ .unsigned_offset = .{ .base = ptr_ra.x(), .offset = 0 } }))
 7261                        else
 7262                            try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
 7263                        try isel.nav_relocs.append(gpa, .{
 7264                            .nav = ty_nav.nav,
 7265                            .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 7266                        });
 7267                        try isel.emit(.adrp(ptr_ra.x(), 0));
 7268                    },
 7269                } else try isel.movImmediate(ptr_ra.x(), isel.pt.navAlignment(ty_nav.nav).forward(0xaaaaaaaaaaaaaaaa));
 7270            }
 7271            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7272        },
 7273        .c_va_arg => {
 7274            const maybe_arg_vi = isel.live_values.fetchRemove(air.inst_index);
 7275            defer if (maybe_arg_vi) |arg_vi| arg_vi.value.deref(isel);
 7276            const ty_op = air.data(air.inst_index).ty_op;
 7277            const ty = ty_op.ty.toType();
 7278            var param_it: CallAbiIterator = .init;
 7279            const param_vi = try param_it.param(isel, ty);
 7280            defer param_vi.?.deref(isel);
 7281            const passed_vi = switch (param_vi.?.parent(isel)) {
 7282                .unallocated => param_vi.?,
 7283                .stack_slot, .value, .constant => unreachable,
 7284                .address => |address_vi| address_vi,
 7285            };
 7286            const passed_size: u5 = @intCast(passed_vi.alignment(isel).forward(passed_vi.size(isel)));
 7287            const passed_is_vector = passed_vi.isVector(isel);
 7288
 7289            const va_list_ptr_vi = try isel.use(ty_op.operand);
 7290            const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
 7291            const offs_ra = try isel.allocIntReg();
 7292            defer isel.freeReg(offs_ra);
 7293            const stack_ra = try isel.allocIntReg();
 7294            defer isel.freeReg(stack_ra);
 7295
 7296            var part_vis: [2]Value.Index = undefined;
 7297            var arg_part_ras: [2]?Register.Alias = @splat(null);
 7298            const parts_len = parts_len: {
 7299                var parts_len: u2 = 0;
 7300                var part_it = passed_vi.parts(isel);
 7301                while (part_it.next()) |part_vi| : (parts_len += 1) {
 7302                    part_vis[parts_len] = part_vi;
 7303                    const arg_vi = maybe_arg_vi orelse continue;
 7304                    const part_offset, const part_size = part_vi.position(isel);
 7305                    var arg_part_it = arg_vi.value.field(ty, part_offset, part_size);
 7306                    const arg_part_vi = try arg_part_it.only(isel);
 7307                    arg_part_ras[parts_len] = try arg_part_vi.?.defReg(isel);
 7308                }
 7309                break :parts_len parts_len;
 7310            };
 7311
 7312            const done_label = isel.instructions.items.len;
 7313            try isel.emit(.str(stack_ra.x(), .{ .unsigned_offset = .{
 7314                .base = va_list_ptr_mat.ra.x(),
 7315                .offset = 0,
 7316            } }));
 7317            try isel.emit(switch (parts_len) {
 7318                else => unreachable,
 7319                1 => if (arg_part_ras[0]) |arg_part_ra| switch (part_vis[0].size(isel)) {
 7320                    else => unreachable,
 7321                    1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .post_index = .{
 7322                        .base = stack_ra.x(),
 7323                        .index = passed_size,
 7324                    } }) else switch (part_vis[0].signedness(isel)) {
 7325                        .signed => .ldrsb(arg_part_ra.w(), .{ .post_index = .{
 7326                            .base = stack_ra.x(),
 7327                            .index = passed_size,
 7328                        } }),
 7329                        .unsigned => .ldrb(arg_part_ra.w(), .{ .post_index = .{
 7330                            .base = stack_ra.x(),
 7331                            .index = passed_size,
 7332                        } }),
 7333                    },
 7334                    2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .post_index = .{
 7335                        .base = stack_ra.x(),
 7336                        .index = passed_size,
 7337                    } }) else switch (part_vis[0].signedness(isel)) {
 7338                        .signed => .ldrsh(arg_part_ra.w(), .{ .post_index = .{
 7339                            .base = stack_ra.x(),
 7340                            .index = passed_size,
 7341                        } }),
 7342                        .unsigned => .ldrh(arg_part_ra.w(), .{ .post_index = .{
 7343                            .base = stack_ra.x(),
 7344                            .index = passed_size,
 7345                        } }),
 7346                    },
 7347                    4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .post_index = .{
 7348                        .base = stack_ra.x(),
 7349                        .index = passed_size,
 7350                    } }),
 7351                    8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .post_index = .{
 7352                        .base = stack_ra.x(),
 7353                        .index = passed_size,
 7354                    } }),
 7355                    16 => .ldr(arg_part_ra.q(), .{ .post_index = .{
 7356                        .base = stack_ra.x(),
 7357                        .index = passed_size,
 7358                    } }),
 7359                } else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
 7360                2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) .ldp(
 7361                    @as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
 7362                    @as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
 7363                    .{ .post_index = .{
 7364                        .base = stack_ra.x(),
 7365                        .index = passed_size,
 7366                    } },
 7367                ) else .add(stack_ra.x(), stack_ra.x(), .{ .immediate = passed_size }),
 7368            });
 7369            try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
 7370                .base = va_list_ptr_mat.ra.x(),
 7371                .offset = 0,
 7372            } }));
 7373            switch (isel.va_list) {
 7374                .other => {},
 7375                .sysv => {
 7376                    const stack_label = isel.instructions.items.len;
 7377                    try isel.emit(.b(
 7378                        @intCast((isel.instructions.items.len + 1 - done_label) << 2),
 7379                    ));
 7380                    switch (parts_len) {
 7381                        else => unreachable,
 7382                        1 => if (arg_part_ras[0]) |arg_part_ra| try isel.emit(switch (part_vis[0].size(isel)) {
 7383                            else => unreachable,
 7384                            1 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.b(), .{ .extended_register = .{
 7385                                .base = stack_ra.x(),
 7386                                .index = offs_ra.w(),
 7387                                .extend = .{ .sxtw = 0 },
 7388                            } }) else switch (part_vis[0].signedness(isel)) {
 7389                                .signed => .ldrsb(arg_part_ra.w(), .{ .extended_register = .{
 7390                                    .base = stack_ra.x(),
 7391                                    .index = offs_ra.w(),
 7392                                    .extend = .{ .sxtw = 0 },
 7393                                } }),
 7394                                .unsigned => .ldrb(arg_part_ra.w(), .{ .extended_register = .{
 7395                                    .base = stack_ra.x(),
 7396                                    .index = offs_ra.w(),
 7397                                    .extend = .{ .sxtw = 0 },
 7398                                } }),
 7399                            },
 7400                            2 => if (arg_part_ra.isVector()) .ldr(arg_part_ra.h(), .{ .extended_register = .{
 7401                                .base = stack_ra.x(),
 7402                                .index = offs_ra.w(),
 7403                                .extend = .{ .sxtw = 0 },
 7404                            } }) else switch (part_vis[0].signedness(isel)) {
 7405                                .signed => .ldrsh(arg_part_ra.w(), .{ .extended_register = .{
 7406                                    .base = stack_ra.x(),
 7407                                    .index = offs_ra.w(),
 7408                                    .extend = .{ .sxtw = 0 },
 7409                                } }),
 7410                                .unsigned => .ldrh(arg_part_ra.w(), .{ .extended_register = .{
 7411                                    .base = stack_ra.x(),
 7412                                    .index = offs_ra.w(),
 7413                                    .extend = .{ .sxtw = 0 },
 7414                                } }),
 7415                            },
 7416                            4 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.s() else arg_part_ra.w(), .{ .extended_register = .{
 7417                                .base = stack_ra.x(),
 7418                                .index = offs_ra.w(),
 7419                                .extend = .{ .sxtw = 0 },
 7420                            } }),
 7421                            8 => .ldr(if (arg_part_ra.isVector()) arg_part_ra.d() else arg_part_ra.x(), .{ .extended_register = .{
 7422                                .base = stack_ra.x(),
 7423                                .index = offs_ra.w(),
 7424                                .extend = .{ .sxtw = 0 },
 7425                            } }),
 7426                            16 => .ldr(arg_part_ra.q(), .{ .extended_register = .{
 7427                                .base = stack_ra.x(),
 7428                                .index = offs_ra.w(),
 7429                                .extend = .{ .sxtw = 0 },
 7430                            } }),
 7431                        }),
 7432                        2 => if (arg_part_ras[0] != null or arg_part_ras[1] != null) {
 7433                            try isel.emit(.ldp(
 7434                                @as(Register.Alias, arg_part_ras[0] orelse .zr).x(),
 7435                                @as(Register.Alias, arg_part_ras[1] orelse .zr).x(),
 7436                                .{ .base = stack_ra.x() },
 7437                            ));
 7438                            try isel.emit(.add(stack_ra.x(), stack_ra.x(), .{ .extended_register = .{
 7439                                .register = offs_ra.w(),
 7440                                .extend = .{ .sxtw = 0 },
 7441                            } }));
 7442                        },
 7443                    }
 7444                    try isel.emit(.ldr(stack_ra.x(), .{ .unsigned_offset = .{
 7445                        .base = va_list_ptr_mat.ra.x(),
 7446                        .offset = if (passed_is_vector) 16 else 8,
 7447                    } }));
 7448                    try isel.emit(.@"b."(
 7449                        .gt,
 7450                        @intCast((isel.instructions.items.len + 1 - stack_label) << 2),
 7451                    ));
 7452                    try isel.emit(.str(stack_ra.w(), .{ .unsigned_offset = .{
 7453                        .base = va_list_ptr_mat.ra.x(),
 7454                        .offset = if (passed_is_vector) 28 else 24,
 7455                    } }));
 7456                    try isel.emit(.adds(stack_ra.w(), offs_ra.w(), .{ .immediate = passed_size }));
 7457                    try isel.emit(.tbz(
 7458                        offs_ra.w(),
 7459                        31,
 7460                        @intCast((isel.instructions.items.len + 1 - stack_label) << 2),
 7461                    ));
 7462                    try isel.emit(.ldr(offs_ra.w(), .{ .unsigned_offset = .{
 7463                        .base = va_list_ptr_mat.ra.x(),
 7464                        .offset = if (passed_is_vector) 28 else 24,
 7465                    } }));
 7466                },
 7467            }
 7468            try va_list_ptr_mat.finish(isel);
 7469            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7470        },
 7471        .c_va_copy => {
 7472            if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| {
 7473                defer va_list_vi.value.deref(isel);
 7474                const ty_op = air.data(air.inst_index).ty_op;
 7475                const va_list_ptr_vi = try isel.use(ty_op.operand);
 7476                const va_list_ptr_mat = try va_list_ptr_vi.matReg(isel);
 7477                _ = try va_list_vi.value.load(isel, ty_op.ty.toType(), va_list_ptr_mat.ra, .{});
 7478                try va_list_ptr_mat.finish(isel);
 7479            }
 7480            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7481        },
 7482        .c_va_end => if (air.next()) |next_air_tag| continue :air_tag next_air_tag,
 7483        .c_va_start => {
 7484            if (isel.live_values.fetchRemove(air.inst_index)) |va_list_vi| {
 7485                defer va_list_vi.value.deref(isel);
 7486                const ty = air.data(air.inst_index).ty;
 7487                switch (isel.va_list) {
 7488                    .other => |va_list| if (try va_list_vi.value.defReg(isel)) |va_list_ra| try isel.emit(.add(
 7489                        va_list_ra.x(),
 7490                        va_list.base.x(),
 7491                        .{ .immediate = @intCast(va_list.offset) },
 7492                    )),
 7493                    .sysv => |va_list| {
 7494                        var vr_offs_it = va_list_vi.value.field(ty, 28, 4);
 7495                        const vr_offs_vi = try vr_offs_it.only(isel);
 7496                        if (try vr_offs_vi.?.defReg(isel)) |vr_offs_ra| try isel.movImmediate(
 7497                            vr_offs_ra.w(),
 7498                            @as(u32, @bitCast(va_list.__vr_offs)),
 7499                        );
 7500                        var gr_offs_it = va_list_vi.value.field(ty, 24, 4);
 7501                        const gr_offs_vi = try gr_offs_it.only(isel);
 7502                        if (try gr_offs_vi.?.defReg(isel)) |gr_offs_ra| try isel.movImmediate(
 7503                            gr_offs_ra.w(),
 7504                            @as(u32, @bitCast(va_list.__gr_offs)),
 7505                        );
 7506                        var vr_top_it = va_list_vi.value.field(ty, 16, 8);
 7507                        const vr_top_vi = try vr_top_it.only(isel);
 7508                        if (try vr_top_vi.?.defReg(isel)) |vr_top_ra| try isel.emit(.add(
 7509                            vr_top_ra.x(),
 7510                            va_list.__vr_top.base.x(),
 7511                            .{ .immediate = @intCast(va_list.__vr_top.offset) },
 7512                        ));
 7513                        var gr_top_it = va_list_vi.value.field(ty, 8, 8);
 7514                        const gr_top_vi = try gr_top_it.only(isel);
 7515                        if (try gr_top_vi.?.defReg(isel)) |gr_top_ra| try isel.emit(.add(
 7516                            gr_top_ra.x(),
 7517                            va_list.__gr_top.base.x(),
 7518                            .{ .immediate = @intCast(va_list.__gr_top.offset) },
 7519                        ));
 7520                        var stack_it = va_list_vi.value.field(ty, 0, 8);
 7521                        const stack_vi = try stack_it.only(isel);
 7522                        if (try stack_vi.?.defReg(isel)) |stack_ra| try isel.emit(.add(
 7523                            stack_ra.x(),
 7524                            va_list.__stack.base.x(),
 7525                            .{ .immediate = @intCast(va_list.__stack.offset) },
 7526                        ));
 7527                    },
 7528                }
 7529            }
 7530            if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
 7531        },
 7532        .work_item_id, .work_group_size, .work_group_id => unreachable,
 7533    }
 7534    assert(air.body_index == 0);
 7535}
 7536
 7537pub fn verify(isel: *Select, check_values: bool) void {
 7538    if (!std.debug.runtime_safety) return;
 7539    assert(isel.blocks.count() == 1 and isel.blocks.keys()[0] == Select.Block.main);
 7540    assert(isel.active_loops.items.len == 0);
 7541    assert(isel.dom_start == 0 and isel.dom_len == 0);
 7542    var live_reg_it = isel.live_registers.iterator();
 7543    while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
 7544        _ => {
 7545            isel.dumpValues(.all);
 7546            unreachable;
 7547        },
 7548        .allocating, .free => {},
 7549    };
 7550    if (check_values) for (isel.values.items) |value| if (value.refs != 0) {
 7551        isel.dumpValues(.only_referenced);
 7552        unreachable;
 7553    };
 7554}
 7555
 7556///           Stack Frame Layout
 7557/// +-+-----------------------------------+
 7558/// |R| allocated stack                   |
 7559/// +-+-----------------------------------+
 7560/// |S| caller frame record               |   +---------------+
 7561/// +-+-----------------------------------+ <-| entry/exit FP |
 7562/// |R| caller frame                      |   +---------------+
 7563/// +-+-----------------------------------+
 7564/// |R| variable incoming stack arguments |   +---------------+
 7565/// +-+-----------------------------------+ <-| __stack       |
 7566/// |S| named incoming stack arguments    |   +---------------+
 7567/// +-+-----------------------------------+ <-| entry/exit SP |
 7568/// |S| incoming gr arguments             |   | __gr_top      |
 7569/// +-+-----------------------------------+   +---------------+
 7570/// |S| alignment gap                     |
 7571/// +-+-----------------------------------+
 7572/// |S| frame record                      |   +----------+
 7573/// +-+-----------------------------------+ <-| FP       |
 7574/// |S| incoming vr arguments             |   | __vr_top |
 7575/// +-+-----------------------------------+   +----------+
 7576/// |L| alignment gap                     |
 7577/// +-+-----------------------------------+
 7578/// |L| callee saved vr area              |
 7579/// +-+-----------------------------------+
 7580/// |L| callee saved gr area              |   +----------------------+
 7581/// +-+-----------------------------------+ <-| prologue/epilogue SP |
 7582/// |R| realignment gap                   |   +----------------------+
 7583/// +-+-----------------------------------+
 7584/// |L| locals                            |
 7585/// +-+-----------------------------------+
 7586/// |S| outgoing stack arguments          |   +----+
 7587/// +-+-----------------------------------+ <-| SP |
 7588/// |R| unallocated stack                 |   +----+
 7589/// +-+-----------------------------------+
 7590/// [S] Size computed by `analyze`, can be used by the body.
 7591/// [L] Size computed by `layout`, can be used by the prologue/epilogue.
 7592/// [R] Size unknown until runtime, can vary from one call to the next.
 7593///
 7594/// Constraints that led to this layout:
 7595///  * FP to __stack/__gr_top/__vr_top must only pass through [S]
 7596///  * SP to outgoing stack arguments/locals must only pass through [S]
 7597///  * entry/exit SP to prologue/epilogue SP must only pass through [S/L]
 7598///  * all save areas must be at a positive offset from prologue/epilogue SP
 7599///  * the entry/exit SP to prologue/epilogue SP distance must
 7600///   - be a multiple of 16 due to hardware restrictions on the value of SP
 7601///   - conform to the limit from the first matching condition in the
 7602///     following list due to instruction encoding limitations
 7603///    1. callee saved gr count >= 2: multiple of 8 of at most 504 bytes
 7604///    2. callee saved vr count >= 2: multiple of 8 of at most 504 bytes
 7605///    3. callee saved gr count >= 1: at most 255 bytes
 7606///    4. callee saved vr count >= 1: at most 255 bytes
 7607///    5. variable incoming vr argument count >= 2: multiple of 16 of at most 1008 bytes
 7608///    6. variable incoming vr argument count >= 1: at most 255 bytes
 7609///    7. have frame record: multiple of 8 of at most 504 bytes
 7610pub fn layout(
 7611    isel: *Select,
 7612    incoming: CallAbiIterator,
 7613    is_sysv_var_args: bool,
 7614    saved_gra_len: u7,
 7615    saved_vra_len: u7,
 7616    mod: *const Package.Module,
 7617) !usize {
 7618    const zcu = isel.pt.zcu;
 7619    const ip = &zcu.intern_pool;
 7620    const nav = ip.getNav(isel.nav_index);
 7621    wip_mir_log.debug("{f}<body>:\n", .{nav.fqn.fmt(ip)});
 7622
 7623    const stack_size: u24 = @intCast(InternPool.Alignment.@"16".forward(isel.stack_size));
 7624
 7625    var saves_buf: [10 + 8 + 8 + 2 + 8]struct {
 7626        class: enum { integer, vector },
 7627        needs_restore: bool,
 7628        register: Register,
 7629        offset: u10,
 7630        size: u5,
 7631    } = undefined;
 7632    const saves, const saves_size, const frame_record_offset = saves: {
 7633        var saves_len: usize = 0;
 7634        var saves_size: u10 = 0;
 7635        var save_ra: Register.Alias = undefined;
 7636
 7637        // callee saved gr area
 7638        save_ra = .r19;
 7639        while (save_ra != .r29) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
 7640            if (!isel.saved_registers.contains(save_ra)) continue;
 7641            saves_size = std.mem.alignForward(u10, saves_size, 8);
 7642            saves_buf[saves_len] = .{
 7643                .class = .integer,
 7644                .needs_restore = true,
 7645                .register = save_ra.x(),
 7646                .offset = saves_size,
 7647                .size = 8,
 7648            };
 7649            saves_len += 1;
 7650            saves_size += 8;
 7651        }
 7652        var deferred_gr = if (saves_size == 8 or (saves_size % 16 != 0 and saved_gra_len % 2 != 0)) gr: {
 7653            saves_len -= 1;
 7654            saves_size -= 8;
 7655            break :gr saves_buf[saves_len].register;
 7656        } else null;
 7657        defer assert(deferred_gr == null);
 7658
 7659        // callee saved vr area
 7660        save_ra = .v8;
 7661        while (save_ra != .v16) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
 7662            if (!isel.saved_registers.contains(save_ra)) continue;
 7663            saves_size = std.mem.alignForward(u10, saves_size, 8);
 7664            saves_buf[saves_len] = .{
 7665                .class = .vector,
 7666                .needs_restore = true,
 7667                .register = save_ra.d(),
 7668                .offset = saves_size,
 7669                .size = 8,
 7670            };
 7671            saves_len += 1;
 7672            saves_size += 8;
 7673        }
 7674        if (deferred_gr != null and saved_gra_len % 2 == 0) {
 7675            saves_size = std.mem.alignForward(u10, saves_size, 8);
 7676            saves_buf[saves_len] = .{
 7677                .class = .integer,
 7678                .needs_restore = true,
 7679                .register = deferred_gr.?,
 7680                .offset = saves_size,
 7681                .size = 8,
 7682            };
 7683            saves_len += 1;
 7684            saves_size += 8;
 7685            deferred_gr = null;
 7686        }
 7687        if (saves_size % 16 != 0 and saved_vra_len % 2 != 0) {
 7688            const prev_save = &saves_buf[saves_len - 1];
 7689            switch (prev_save.class) {
 7690                .integer => {},
 7691                .vector => {
 7692                    prev_save.register = prev_save.register.alias.q();
 7693                    prev_save.size = 16;
 7694                    saves_size += 8;
 7695                },
 7696            }
 7697        }
 7698
 7699        // incoming vr arguments
 7700        save_ra = if (mod.strip) incoming.nsrn else CallAbiIterator.nsrn_start;
 7701        while (save_ra != if (is_sysv_var_args) CallAbiIterator.nsrn_end else incoming.nsrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
 7702            saves_size = std.mem.alignForward(u10, saves_size, 16);
 7703            saves_buf[saves_len] = .{
 7704                .class = .vector,
 7705                .needs_restore = false,
 7706                .register = save_ra.q(),
 7707                .offset = saves_size,
 7708                .size = 16,
 7709            };
 7710            saves_len += 1;
 7711            saves_size += 16;
 7712        }
 7713
 7714        // frame record
 7715        saves_size = std.mem.alignForward(u10, saves_size, 16);
 7716        const frame_record_offset = saves_size;
 7717        saves_buf[saves_len] = .{
 7718            .class = .integer,
 7719            .needs_restore = true,
 7720            .register = .fp,
 7721            .offset = saves_size,
 7722            .size = 8,
 7723        };
 7724        saves_len += 1;
 7725        saves_size += 8;
 7726
 7727        saves_size = std.mem.alignForward(u10, saves_size, 8);
 7728        saves_buf[saves_len] = .{
 7729            .class = .integer,
 7730            .needs_restore = true,
 7731            .register = .lr,
 7732            .offset = saves_size,
 7733            .size = 8,
 7734        };
 7735        saves_len += 1;
 7736        saves_size += 8;
 7737
 7738        // incoming gr arguments
 7739        if (deferred_gr) |gr| {
 7740            saves_size = std.mem.alignForward(u10, saves_size, 8);
 7741            saves_buf[saves_len] = .{
 7742                .class = .integer,
 7743                .needs_restore = true,
 7744                .register = gr,
 7745                .offset = saves_size,
 7746                .size = 8,
 7747            };
 7748            saves_len += 1;
 7749            saves_size += 8;
 7750            deferred_gr = null;
 7751        } else switch (@as(u1, @truncate(saved_gra_len))) {
 7752            0 => {},
 7753            1 => saves_size += 8,
 7754        }
 7755        save_ra = if (mod.strip) incoming.ngrn else CallAbiIterator.ngrn_start;
 7756        while (save_ra != if (is_sysv_var_args) CallAbiIterator.ngrn_end else incoming.ngrn) : (save_ra = @enumFromInt(@intFromEnum(save_ra) + 1)) {
 7757            saves_size = std.mem.alignForward(u10, saves_size, 8);
 7758            saves_buf[saves_len] = .{
 7759                .class = .integer,
 7760                .needs_restore = false,
 7761                .register = save_ra.x(),
 7762                .offset = saves_size,
 7763                .size = 8,
 7764            };
 7765            saves_len += 1;
 7766            saves_size += 8;
 7767        }
 7768
 7769        assert(InternPool.Alignment.@"16".check(saves_size));
 7770        break :saves .{ saves_buf[0..saves_len], saves_size, frame_record_offset };
 7771    };
 7772
 7773    {
 7774        wip_mir_log.debug("{f}<prologue>:", .{nav.fqn.fmt(ip)});
 7775        var save_index: usize = 0;
 7776        while (save_index < saves.len) if (save_index + 2 <= saves.len and
 7777            saves[save_index + 0].class == saves[save_index + 1].class and
 7778            saves[save_index + 0].size == saves[save_index + 1].size and
 7779            saves[save_index + 0].offset + saves[save_index + 0].size == saves[save_index + 1].offset)
 7780        {
 7781            try isel.emit(.stp(
 7782                saves[save_index + 0].register,
 7783                saves[save_index + 1].register,
 7784                switch (saves[save_index + 0].offset) {
 7785                    0 => .{ .pre_index = .{
 7786                        .base = .sp,
 7787                        .index = @intCast(-@as(i11, saves_size)),
 7788                    } },
 7789                    else => |offset| .{ .signed_offset = .{
 7790                        .base = .sp,
 7791                        .offset = @intCast(offset),
 7792                    } },
 7793                },
 7794            ));
 7795            save_index += 2;
 7796        } else {
 7797            try isel.emit(.str(
 7798                saves[save_index].register,
 7799                switch (saves[save_index].offset) {
 7800                    0 => .{ .pre_index = .{
 7801                        .base = .sp,
 7802                        .index = @intCast(-@as(i11, saves_size)),
 7803                    } },
 7804                    else => |offset| .{ .unsigned_offset = .{
 7805                        .base = .sp,
 7806                        .offset = @intCast(offset),
 7807                    } },
 7808                },
 7809            ));
 7810            save_index += 1;
 7811        };
 7812
 7813        try isel.emit(.add(.fp, .sp, .{ .immediate = frame_record_offset }));
 7814        const scratch_reg: Register = if (isel.stack_align == .@"16")
 7815            .sp
 7816        else if (stack_size == 0 and frame_record_offset == 0)
 7817            .fp
 7818        else
 7819            .ip0;
 7820        const stack_size_lo: u12 = @truncate(stack_size >> 0);
 7821        const stack_size_hi: u12 = @truncate(stack_size >> 12);
 7822        if (mod.stack_check) {
 7823            if (stack_size_hi > 2) {
 7824                try isel.movImmediate(.ip1, stack_size_hi);
 7825                const loop_label = isel.instructions.items.len;
 7826                try isel.emit(.sub(.sp, .sp, .{
 7827                    .shifted_immediate = .{ .immediate = 1, .lsl = .@"12" },
 7828                }));
 7829                try isel.emit(.sub(.ip1, .ip1, .{ .immediate = 1 }));
 7830                try isel.emit(.ldr(.xzr, .{ .base = .sp }));
 7831                try isel.emit(.cbnz(.ip1, -@as(i21, @intCast(
 7832                    (isel.instructions.items.len - loop_label) << 2,
 7833                ))));
 7834            } else for (0..stack_size_hi) |_| {
 7835                try isel.emit(.sub(.sp, .sp, .{
 7836                    .shifted_immediate = .{ .immediate = 1, .lsl = .@"12" },
 7837                }));
 7838                try isel.emit(.ldr(.xzr, .{ .base = .sp }));
 7839            }
 7840            if (stack_size_lo > 0) try isel.emit(.sub(
 7841                scratch_reg,
 7842                .sp,
 7843                .{ .immediate = stack_size_lo },
 7844            )) else if (scratch_reg.alias == Register.Alias.ip0)
 7845                try isel.emit(.add(scratch_reg, .sp, .{ .immediate = 0 }));
 7846        } else {
 7847            if (stack_size_hi > 0) try isel.emit(.sub(scratch_reg, .sp, .{
 7848                .shifted_immediate = .{ .immediate = stack_size_hi, .lsl = .@"12" },
 7849            }));
 7850            if (stack_size_lo > 0) try isel.emit(.sub(
 7851                scratch_reg,
 7852                if (stack_size_hi > 0) scratch_reg else .sp,
 7853                .{ .immediate = stack_size_lo },
 7854            )) else if (scratch_reg.alias == Register.Alias.ip0 and stack_size_hi == 0)
 7855                try isel.emit(.add(scratch_reg, .sp, .{ .immediate = 0 }));
 7856        }
 7857        if (isel.stack_align != .@"16") try isel.emit(.@"and"(.sp, scratch_reg, .{ .immediate = .{
 7858            .N = .doubleword,
 7859            .immr = -%isel.stack_align.toLog2Units(),
 7860            .imms = ~isel.stack_align.toLog2Units(),
 7861        } }));
 7862        wip_mir_log.debug("", .{});
 7863    }
 7864
 7865    const epilogue = isel.instructions.items.len;
 7866    if (isel.returns) {
 7867        try isel.emit(.ret(.lr));
 7868        var save_index: usize = 0;
 7869        var first_offset: ?u10 = null;
 7870        while (save_index < saves.len) {
 7871            if (save_index + 2 <= saves.len and saves[save_index + 1].needs_restore and
 7872                saves[save_index + 0].class == saves[save_index + 1].class and
 7873                saves[save_index + 0].size == saves[save_index + 1].size and
 7874                saves[save_index + 0].offset + saves[save_index + 0].size == saves[save_index + 1].offset)
 7875            {
 7876                try isel.emit(.ldp(
 7877                    saves[save_index + 0].register,
 7878                    saves[save_index + 1].register,
 7879                    if (first_offset) |offset| .{ .signed_offset = .{
 7880                        .base = .sp,
 7881                        .offset = @intCast(saves[save_index + 0].offset - offset),
 7882                    } } else form: {
 7883                        first_offset = @intCast(saves[save_index + 0].offset);
 7884                        break :form .{ .post_index = .{
 7885                            .base = .sp,
 7886                            .index = @intCast(saves_size - first_offset.?),
 7887                        } };
 7888                    },
 7889                ));
 7890                save_index += 2;
 7891            } else if (saves[save_index].needs_restore) {
 7892                try isel.emit(.ldr(
 7893                    saves[save_index].register,
 7894                    if (first_offset) |offset| .{ .unsigned_offset = .{
 7895                        .base = .sp,
 7896                        .offset = saves[save_index + 0].offset - offset,
 7897                    } } else form: {
 7898                        const offset = saves[save_index + 0].offset;
 7899                        first_offset = offset;
 7900                        break :form .{ .post_index = .{
 7901                            .base = .sp,
 7902                            .index = @intCast(saves_size - offset),
 7903                        } };
 7904                    },
 7905                ));
 7906                save_index += 1;
 7907            } else save_index += 1;
 7908        }
 7909        const offset = stack_size + first_offset.?;
 7910        const offset_lo: u12 = @truncate(offset >> 0);
 7911        const offset_hi: u12 = @truncate(offset >> 12);
 7912        if (isel.stack_align != .@"16" or (offset_lo > 0 and offset_hi > 0)) {
 7913            const fp_offset = @as(i11, first_offset.?) - frame_record_offset;
 7914            try isel.emit(if (fp_offset >= 0)
 7915                .add(.sp, .fp, .{ .immediate = @intCast(fp_offset) })
 7916            else
 7917                .sub(.sp, .fp, .{ .immediate = @intCast(-fp_offset) }));
 7918        } else {
 7919            if (offset_hi > 0) try isel.emit(.add(.sp, .sp, .{
 7920                .shifted_immediate = .{ .immediate = offset_hi, .lsl = .@"12" },
 7921            }));
 7922            if (offset_lo > 0) try isel.emit(.add(.sp, .sp, .{
 7923                .immediate = offset_lo,
 7924            }));
 7925        }
 7926        wip_mir_log.debug("{f}<epilogue>:\n", .{nav.fqn.fmt(ip)});
 7927    }
 7928    return epilogue;
 7929}
 7930
 7931fn fmtDom(isel: *Select, inst: Air.Inst.Index, start: u32, len: u32) struct {
 7932    isel: *Select,
 7933    inst: Air.Inst.Index,
 7934    start: u32,
 7935    len: u32,
 7936    pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
 7937        try writer.print("%{d} -> {{", .{@intFromEnum(data.inst)});
 7938        var first = true;
 7939        for (data.isel.blocks.keys()[0..data.len], 0..) |block_inst_index, dom_index| {
 7940            if (@as(u1, @truncate(data.isel.dom.items[
 7941                data.start + dom_index / @bitSizeOf(DomInt)
 7942            ] >> @truncate(dom_index))) == 0) continue;
 7943            if (first) {
 7944                first = false;
 7945            } else {
 7946                try writer.writeByte(',');
 7947            }
 7948            switch (block_inst_index) {
 7949                Block.main => try writer.writeAll(" %main"),
 7950                else => try writer.print(" %{d}", .{@intFromEnum(block_inst_index)}),
 7951            }
 7952        }
 7953        if (!first) try writer.writeByte(' ');
 7954        try writer.writeByte('}');
 7955    }
 7956} {
 7957    return .{ .isel = isel, .inst = inst, .start = start, .len = len };
 7958}
 7959
 7960fn fmtLoopLive(isel: *Select, loop_inst: Air.Inst.Index) struct {
 7961    isel: *Select,
 7962    inst: Air.Inst.Index,
 7963    pub fn format(data: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void {
 7964        const loops = data.isel.loops.values();
 7965        const loop_index = data.isel.loops.getIndex(data.inst).?;
 7966        const live_insts =
 7967            data.isel.loop_live.list.items[loops[loop_index].live..loops[loop_index + 1].live];
 7968
 7969        try writer.print("%{d} <- {{", .{@intFromEnum(data.inst)});
 7970        var first = true;
 7971        for (live_insts) |live_inst| {
 7972            if (first) {
 7973                first = false;
 7974            } else {
 7975                try writer.writeByte(',');
 7976            }
 7977            try writer.print(" %{d}", .{@intFromEnum(live_inst)});
 7978        }
 7979        if (!first) try writer.writeByte(' ');
 7980        try writer.writeByte('}');
 7981    }
 7982} {
 7983    return .{ .isel = isel, .inst = loop_inst };
 7984}
 7985
 7986fn fmtType(isel: *Select, ty: ZigType) ZigType.Formatter {
 7987    return ty.fmt(isel.pt);
 7988}
 7989
 7990fn fmtConstant(isel: *Select, constant: Constant) @typeInfo(@TypeOf(Constant.fmtValue)).@"fn".return_type.? {
 7991    return constant.fmtValue(isel.pt);
 7992}
 7993
 7994fn block(
 7995    isel: *Select,
 7996    air_inst_index: Air.Inst.Index,
 7997    res_ty: ZigType,
 7998    air_body: []const Air.Inst.Index,
 7999) !void {
 8000    if (res_ty.toIntern() != .noreturn_type) {
 8001        isel.blocks.putAssumeCapacityNoClobber(air_inst_index, .{
 8002            .live_registers = isel.live_registers,
 8003            .target_label = @intCast(isel.instructions.items.len),
 8004        });
 8005    }
 8006    try isel.body(air_body);
 8007    if (res_ty.toIntern() != .noreturn_type) {
 8008        const block_entry = isel.blocks.pop().?;
 8009        assert(block_entry.key == air_inst_index);
 8010        if (isel.live_values.fetchRemove(air_inst_index)) |result_vi| result_vi.value.deref(isel);
 8011    }
 8012}
 8013
 8014fn emit(isel: *Select, instruction: codegen.aarch64.encoding.Instruction) !void {
 8015    wip_mir_log.debug("  | {f}", .{instruction});
 8016    try isel.instructions.append(isel.pt.zcu.gpa, instruction);
 8017}
 8018
 8019fn emitPanic(isel: *Select, panic_id: Zcu.SimplePanicId) !void {
 8020    const zcu = isel.pt.zcu;
 8021    try isel.nav_relocs.append(zcu.gpa, .{
 8022        .nav = switch (zcu.intern_pool.indexToKey(zcu.builtin_decl_values.get(panic_id.toBuiltin()))) {
 8023            else => unreachable,
 8024            inline .@"extern", .func => |func| func.owner_nav,
 8025        },
 8026        .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 8027    });
 8028    try isel.emit(.bl(0));
 8029}
 8030
 8031fn emitLiteral(isel: *Select, bytes: []const u8) !void {
 8032    const words: []align(1) const u32 = @ptrCast(bytes);
 8033    const literals = try isel.literals.addManyAsSlice(isel.pt.zcu.gpa, words.len);
 8034    switch (isel.target.cpu.arch.endian()) {
 8035        .little => @memcpy(literals, words),
 8036        .big => for (words, 0..) |word, word_index| {
 8037            literals[literals.len - 1 - word_index] = @byteSwap(word);
 8038        },
 8039    }
 8040}
 8041
 8042fn fail(isel: *Select, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
 8043    @branchHint(.cold);
 8044    return isel.pt.zcu.codegenFail(isel.nav_index, format, args);
 8045}
 8046
 8047/// dst = src
 8048fn movImmediate(isel: *Select, dst_reg: Register, src_imm: u64) !void {
 8049    const sf = dst_reg.format.general;
 8050    if (src_imm == 0) {
 8051        const zr: Register = switch (sf) {
 8052            .word => .wzr,
 8053            .doubleword => .xzr,
 8054        };
 8055        return isel.emit(.orr(dst_reg, zr, .{ .register = zr }));
 8056    }
 8057
 8058    const Part = u16;
 8059    const min_part: Part = std.math.minInt(Part);
 8060    const max_part: Part = std.math.maxInt(Part);
 8061
 8062    const parts: [4]Part = @bitCast(switch (sf) {
 8063        .word => @as(u32, @intCast(src_imm)),
 8064        .doubleword => @as(u64, @intCast(src_imm)),
 8065    });
 8066    const width: u7 = switch (sf) {
 8067        .word => 32,
 8068        .doubleword => 64,
 8069    };
 8070    const parts_len: u3 = @intCast(@divExact(width, @bitSizeOf(Part)));
 8071    var equal_min_count: u3 = 0;
 8072    var equal_max_count: u3 = 0;
 8073    for (parts[0..parts_len]) |part| {
 8074        equal_min_count += @intFromBool(part == min_part);
 8075        equal_max_count += @intFromBool(part == max_part);
 8076    }
 8077
 8078    const equal_fill_count, const fill_part: Part = if (equal_min_count >= equal_max_count)
 8079        .{ equal_min_count, min_part }
 8080    else
 8081        .{ equal_max_count, max_part };
 8082    var remaining_parts = @max(parts_len - equal_fill_count, 1);
 8083
 8084    if (remaining_parts > 1) {
 8085        var elem_width: u8 = 2;
 8086        while (elem_width <= width) : (elem_width <<= 1) {
 8087            const emask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - elem_width);
 8088            const rmask = @divExact(@as(u64, switch (sf) {
 8089                .word => std.math.maxInt(u32),
 8090                .doubleword => std.math.maxInt(u64),
 8091            }), emask);
 8092            const elem = src_imm & emask;
 8093            if (src_imm != elem * rmask) continue;
 8094            const imask: u64 = @bitCast(@as(i64, @bitCast(elem << 63)) >> 63);
 8095            const lsb0 = elem ^ (imask & emask);
 8096            const lsb1 = (lsb0 - 1) | lsb0;
 8097            if ((lsb1 +% 1) & lsb1 == 0) {
 8098                const lo: u6 = @intCast(@ctz(lsb0));
 8099                const hi: u6 = @intCast(@clz(lsb0) - (64 - elem_width));
 8100                const mid: u6 = @intCast(elem_width - lo - hi);
 8101                const smask: u6 = @truncate(imask);
 8102                const mid_masked = mid & ~smask;
 8103                return isel.emit(.orr(
 8104                    dst_reg,
 8105                    switch (sf) {
 8106                        .word => .wzr,
 8107                        .doubleword => .xzr,
 8108                    },
 8109                    .{ .immediate = .{
 8110                        .N = @enumFromInt(elem_width >> 6),
 8111                        .immr = hi + mid_masked,
 8112                        .imms = ((((lo + hi) & smask) | mid_masked) - 1) | -%@as(u6, @truncate(elem_width)) << 1,
 8113                    } },
 8114                ));
 8115            }
 8116        }
 8117    }
 8118
 8119    var part_index = parts_len;
 8120    while (part_index > 0) {
 8121        part_index -= 1;
 8122        if (part_index >= remaining_parts and parts[part_index] == fill_part) continue;
 8123        remaining_parts -= 1;
 8124        try isel.emit(if (remaining_parts > 0) .movk(
 8125            dst_reg,
 8126            parts[part_index],
 8127            .{ .lsl = @enumFromInt(part_index) },
 8128        ) else switch (fill_part) {
 8129            else => unreachable,
 8130            min_part => .movz(
 8131                dst_reg,
 8132                parts[part_index],
 8133                .{ .lsl = @enumFromInt(part_index) },
 8134            ),
 8135            max_part => .movn(
 8136                dst_reg,
 8137                ~parts[part_index],
 8138                .{ .lsl = @enumFromInt(part_index) },
 8139            ),
 8140        });
 8141    }
 8142    assert(remaining_parts == 0);
 8143}
 8144
 8145/// elem_ptr = base +- elem_size * index
 8146/// elem_ptr, base, and index may alias
 8147fn elemPtr(
 8148    isel: *Select,
 8149    elem_ptr_ra: Register.Alias,
 8150    base_ra: Register.Alias,
 8151    op: codegen.aarch64.encoding.Instruction.AddSubtractOp,
 8152    elem_size: u64,
 8153    index_vi: Value.Index,
 8154) !void {
 8155    const index_mat = try index_vi.matReg(isel);
 8156    switch (@popCount(elem_size)) {
 8157        0 => unreachable,
 8158        1 => try isel.emit(switch (op) {
 8159            .add => switch (base_ra) {
 8160                else => .add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8161                    .register = index_mat.ra.x(),
 8162                    .shift = .{ .lsl = @intCast(@ctz(elem_size)) },
 8163                } }),
 8164                .zr => switch (@ctz(elem_size)) {
 8165                    0 => .orr(elem_ptr_ra.x(), .xzr, .{ .register = index_mat.ra.x() }),
 8166                    else => |shift| .ubfm(elem_ptr_ra.x(), index_mat.ra.x(), .{
 8167                        .N = .doubleword,
 8168                        .immr = @intCast(64 - shift),
 8169                        .imms = @intCast(63 - shift),
 8170                    }),
 8171                },
 8172            },
 8173            .sub => .sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8174                .register = index_mat.ra.x(),
 8175                .shift = .{ .lsl = @intCast(@ctz(elem_size)) },
 8176            } }),
 8177        }),
 8178        2 => {
 8179            const shift: u6 = @intCast(@ctz(elem_size));
 8180            const temp_ra, const free_temp_ra = temp_ra: switch (op) {
 8181                .add => switch (base_ra) {
 8182                    else => {
 8183                        const temp_ra = try isel.allocIntReg();
 8184                        errdefer isel.freeReg(temp_ra);
 8185                        try isel.emit(.add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8186                            .register = temp_ra.x(),
 8187                            .shift = .{ .lsl = shift },
 8188                        } }));
 8189                        break :temp_ra .{ temp_ra, true };
 8190                    },
 8191                    .zr => {
 8192                        if (shift > 0) try isel.emit(.ubfm(elem_ptr_ra.x(), elem_ptr_ra.x(), .{
 8193                            .N = .doubleword,
 8194                            .immr = -%shift,
 8195                            .imms = ~shift,
 8196                        }));
 8197                        break :temp_ra .{ elem_ptr_ra, false };
 8198                    },
 8199                },
 8200                .sub => {
 8201                    const temp_ra = try isel.allocIntReg();
 8202                    errdefer isel.freeReg(temp_ra);
 8203                    try isel.emit(.sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8204                        .register = temp_ra.x(),
 8205                        .shift = .{ .lsl = shift },
 8206                    } }));
 8207                    break :temp_ra .{ temp_ra, true };
 8208                },
 8209            };
 8210            defer if (free_temp_ra) isel.freeReg(temp_ra);
 8211            try isel.emit(.add(temp_ra.x(), index_mat.ra.x(), .{ .shifted_register = .{
 8212                .register = index_mat.ra.x(),
 8213                .shift = .{ .lsl = @intCast(63 - @clz(elem_size) - shift) },
 8214            } }));
 8215        },
 8216        else => {
 8217            const elem_size_lsb1 = (elem_size - 1) | elem_size;
 8218            if ((elem_size_lsb1 +% 1) & elem_size_lsb1 == 0) {
 8219                const shift: u6 = @intCast(@ctz(elem_size));
 8220                const temp_ra = temp_ra: switch (op) {
 8221                    .add => {
 8222                        const temp_ra = try isel.allocIntReg();
 8223                        errdefer isel.freeReg(temp_ra);
 8224                        try isel.emit(.sub(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8225                            .register = temp_ra.x(),
 8226                            .shift = .{ .lsl = shift },
 8227                        } }));
 8228                        break :temp_ra temp_ra;
 8229                    },
 8230                    .sub => switch (base_ra) {
 8231                        else => {
 8232                            const temp_ra = try isel.allocIntReg();
 8233                            errdefer isel.freeReg(temp_ra);
 8234                            try isel.emit(.add(elem_ptr_ra.x(), base_ra.x(), .{ .shifted_register = .{
 8235                                .register = temp_ra.x(),
 8236                                .shift = .{ .lsl = shift },
 8237                            } }));
 8238                            break :temp_ra temp_ra;
 8239                        },
 8240                        .zr => {
 8241                            if (shift > 0) try isel.emit(.ubfm(elem_ptr_ra.x(), elem_ptr_ra.x(), .{
 8242                                .N = .doubleword,
 8243                                .immr = -%shift,
 8244                                .imms = ~shift,
 8245                            }));
 8246                            break :temp_ra elem_ptr_ra;
 8247                        },
 8248                    },
 8249                };
 8250                defer if (temp_ra != elem_ptr_ra) isel.freeReg(temp_ra);
 8251                try isel.emit(.sub(temp_ra.x(), index_mat.ra.x(), .{ .shifted_register = .{
 8252                    .register = index_mat.ra.x(),
 8253                    .shift = .{ .lsl = @intCast(64 - @clz(elem_size) - shift) },
 8254                } }));
 8255            } else {
 8256                try isel.emit(switch (op) {
 8257                    .add => .madd(elem_ptr_ra.x(), index_mat.ra.x(), elem_ptr_ra.x(), base_ra.x()),
 8258                    .sub => .msub(elem_ptr_ra.x(), index_mat.ra.x(), elem_ptr_ra.x(), base_ra.x()),
 8259                });
 8260                try isel.movImmediate(elem_ptr_ra.x(), elem_size);
 8261            }
 8262        },
 8263    }
 8264    try index_mat.finish(isel);
 8265}
 8266
 8267fn clzLimb(
 8268    isel: *Select,
 8269    res_ra: Register.Alias,
 8270    src_int_info: std.builtin.Type.Int,
 8271    src_ra: Register.Alias,
 8272) !void {
 8273    switch (src_int_info.bits) {
 8274        else => unreachable,
 8275        1...31 => |bits| {
 8276            try isel.emit(.sub(res_ra.w(), res_ra.w(), .{
 8277                .immediate = @intCast(32 - bits),
 8278            }));
 8279            switch (src_int_info.signedness) {
 8280                .signed => {
 8281                    try isel.emit(.clz(res_ra.w(), res_ra.w()));
 8282                    try isel.emit(.ubfm(res_ra.w(), src_ra.w(), .{
 8283                        .N = .word,
 8284                        .immr = 0,
 8285                        .imms = @intCast(bits - 1),
 8286                    }));
 8287                },
 8288                .unsigned => try isel.emit(.clz(res_ra.w(), src_ra.w())),
 8289            }
 8290        },
 8291        32 => try isel.emit(.clz(res_ra.w(), src_ra.w())),
 8292        33...63 => |bits| {
 8293            try isel.emit(.sub(res_ra.w(), res_ra.w(), .{
 8294                .immediate = @intCast(64 - bits),
 8295            }));
 8296            switch (src_int_info.signedness) {
 8297                .signed => {
 8298                    try isel.emit(.clz(res_ra.x(), res_ra.x()));
 8299                    try isel.emit(.ubfm(res_ra.x(), src_ra.x(), .{
 8300                        .N = .doubleword,
 8301                        .immr = 0,
 8302                        .imms = @intCast(bits - 1),
 8303                    }));
 8304                },
 8305                .unsigned => try isel.emit(.clz(res_ra.x(), src_ra.x())),
 8306            }
 8307        },
 8308        64 => try isel.emit(.clz(res_ra.x(), src_ra.x())),
 8309    }
 8310}
 8311
 8312fn ctzLimb(
 8313    isel: *Select,
 8314    res_ra: Register.Alias,
 8315    src_int_info: std.builtin.Type.Int,
 8316    src_ra: Register.Alias,
 8317) !void {
 8318    switch (src_int_info.bits) {
 8319        else => unreachable,
 8320        1...31 => |bits| {
 8321            try isel.emit(.clz(res_ra.w(), res_ra.w()));
 8322            try isel.emit(.rbit(res_ra.w(), res_ra.w()));
 8323            try isel.emit(.orr(res_ra.w(), src_ra.w(), .{ .immediate = .{
 8324                .N = .word,
 8325                .immr = @intCast(32 - bits),
 8326                .imms = @intCast(32 - bits - 1),
 8327            } }));
 8328        },
 8329        32 => {
 8330            try isel.emit(.clz(res_ra.w(), res_ra.w()));
 8331            try isel.emit(.rbit(res_ra.w(), src_ra.w()));
 8332        },
 8333        33...63 => |bits| {
 8334            try isel.emit(.clz(res_ra.x(), res_ra.x()));
 8335            try isel.emit(.rbit(res_ra.x(), res_ra.x()));
 8336            try isel.emit(.orr(res_ra.x(), src_ra.x(), .{ .immediate = .{
 8337                .N = .doubleword,
 8338                .immr = @intCast(64 - bits),
 8339                .imms = @intCast(64 - bits - 1),
 8340            } }));
 8341        },
 8342        64 => {
 8343            try isel.emit(.clz(res_ra.x(), res_ra.x()));
 8344            try isel.emit(.rbit(res_ra.x(), src_ra.x()));
 8345        },
 8346    }
 8347}
 8348
 8349fn cmp(
 8350    isel: *Select,
 8351    res_ra: Register.Alias,
 8352    ty: ZigType,
 8353    orig_lhs_vi: Value.Index,
 8354    op: std.math.CompareOperator,
 8355    orig_rhs_vi: Value.Index,
 8356) !struct { cset_label: usize } {
 8357    var lhs_vi = orig_lhs_vi;
 8358    var rhs_vi = orig_rhs_vi;
 8359    if (!ty.isRuntimeFloat()) {
 8360        const int_info: std.builtin.Type.Int = if (ty.toIntern() == .bool_type)
 8361            .{ .signedness = .unsigned, .bits = 1 }
 8362        else if (ty.isAbiInt(isel.pt.zcu))
 8363            ty.intInfo(isel.pt.zcu)
 8364        else if (ty.isPtrAtRuntime(isel.pt.zcu))
 8365            .{ .signedness = .unsigned, .bits = 64 }
 8366        else
 8367            return isel.fail("bad cmp_{t} {f}", .{ op, isel.fmtType(ty) });
 8368        if (int_info.bits > 256) return isel.fail("too big cmp_{t} {f}", .{ op, isel.fmtType(ty) });
 8369        try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(cond: switch (op) {
 8370            .lt => switch (int_info.signedness) {
 8371                .signed => .lt,
 8372                .unsigned => .lo,
 8373            },
 8374            .lte => switch (int_info.bits) {
 8375                else => unreachable,
 8376                1...64 => switch (int_info.signedness) {
 8377                    .signed => .le,
 8378                    .unsigned => .ls,
 8379                },
 8380                65...128 => {
 8381                    std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
 8382                    continue :cond .gte;
 8383                },
 8384            },
 8385            .eq => .eq,
 8386            .gte => switch (int_info.signedness) {
 8387                .signed => .ge,
 8388                .unsigned => .hs,
 8389            },
 8390            .gt => switch (int_info.bits) {
 8391                else => unreachable,
 8392                1...64 => switch (int_info.signedness) {
 8393                    .signed => .gt,
 8394                    .unsigned => .hi,
 8395                },
 8396                65...128 => {
 8397                    std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
 8398                    continue :cond .lt;
 8399                },
 8400            },
 8401            .neq => .ne,
 8402        })));
 8403        const cset_label = isel.instructions.items.len;
 8404
 8405        var part_offset = lhs_vi.size(isel);
 8406        while (part_offset > 0) {
 8407            const part_size = @min(part_offset, 8);
 8408            part_offset -= part_size;
 8409            var lhs_part_it = lhs_vi.field(ty, part_offset, part_size);
 8410            const lhs_part_vi = try lhs_part_it.only(isel);
 8411            const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
 8412            var rhs_part_it = rhs_vi.field(ty, part_offset, part_size);
 8413            const rhs_part_vi = try rhs_part_it.only(isel);
 8414            const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
 8415            try isel.emit(switch (part_size) {
 8416                else => unreachable,
 8417                1...4 => switch (part_offset) {
 8418                    0 => .subs(.wzr, lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 8419                    else => switch (op) {
 8420                        .lt, .lte, .gte, .gt => .sbcs(
 8421                            .wzr,
 8422                            lhs_part_mat.ra.w(),
 8423                            rhs_part_mat.ra.w(),
 8424                        ),
 8425                        .eq, .neq => .ccmp(
 8426                            lhs_part_mat.ra.w(),
 8427                            .{ .register = rhs_part_mat.ra.w() },
 8428                            .{ .n = false, .z = false, .c = false, .v = false },
 8429                            .eq,
 8430                        ),
 8431                    },
 8432                },
 8433                5...8 => switch (part_offset) {
 8434                    0 => .subs(.xzr, lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 8435                    else => switch (op) {
 8436                        .lt, .lte, .gte, .gt => .sbcs(
 8437                            .xzr,
 8438                            lhs_part_mat.ra.x(),
 8439                            rhs_part_mat.ra.x(),
 8440                        ),
 8441                        .eq, .neq => .ccmp(
 8442                            lhs_part_mat.ra.x(),
 8443                            .{ .register = rhs_part_mat.ra.x() },
 8444                            .{ .n = false, .z = false, .c = false, .v = false },
 8445                            .eq,
 8446                        ),
 8447                    },
 8448                },
 8449            });
 8450            try rhs_part_mat.finish(isel);
 8451            try lhs_part_mat.finish(isel);
 8452        }
 8453        return .{ .cset_label = cset_label };
 8454    }
 8455    switch (ty.floatBits(isel.target)) {
 8456        else => unreachable,
 8457        16, 32, 64 => |bits| {
 8458            const need_fcvt = switch (bits) {
 8459                else => unreachable,
 8460                16 => !isel.target.cpu.has(.aarch64, .fullfp16),
 8461                32, 64 => false,
 8462            };
 8463            try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(switch (op) {
 8464                .lt => .lo,
 8465                .lte => .ls,
 8466                .eq => .eq,
 8467                .gte => .ge,
 8468                .gt => .gt,
 8469                .neq => .ne,
 8470            })));
 8471            const cset_label = isel.instructions.items.len;
 8472
 8473            const lhs_mat = try lhs_vi.matReg(isel);
 8474            const rhs_mat = try rhs_vi.matReg(isel);
 8475            const lhs_ra = if (need_fcvt) try isel.allocVecReg() else lhs_mat.ra;
 8476            defer if (need_fcvt) isel.freeReg(lhs_ra);
 8477            const rhs_ra = if (need_fcvt) try isel.allocVecReg() else rhs_mat.ra;
 8478            defer if (need_fcvt) isel.freeReg(rhs_ra);
 8479            try isel.emit(bits: switch (bits) {
 8480                else => unreachable,
 8481                16 => if (need_fcvt)
 8482                    continue :bits 32
 8483                else
 8484                    .fcmp(lhs_ra.h(), .{ .register = rhs_ra.h() }),
 8485                32 => .fcmp(lhs_ra.s(), .{ .register = rhs_ra.s() }),
 8486                64 => .fcmp(lhs_ra.d(), .{ .register = rhs_ra.d() }),
 8487            });
 8488            if (need_fcvt) {
 8489                try isel.emit(.fcvt(rhs_ra.s(), rhs_mat.ra.h()));
 8490                try isel.emit(.fcvt(lhs_ra.s(), lhs_mat.ra.h()));
 8491            }
 8492            try rhs_mat.finish(isel);
 8493            try lhs_mat.finish(isel);
 8494            return .{ .cset_label = cset_label };
 8495        },
 8496        80, 128 => |bits| {
 8497            try call.prepareReturn(isel);
 8498            try call.returnFill(isel, .r0);
 8499            try isel.emit(.csinc(res_ra.w(), .wzr, .wzr, .invert(cond: switch (op) {
 8500                .lt => .lt,
 8501                .lte => .le,
 8502                .eq => .eq,
 8503                .gte => {
 8504                    std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
 8505                    continue :cond .lte;
 8506                },
 8507                .gt => {
 8508                    std.mem.swap(Value.Index, &lhs_vi, &rhs_vi);
 8509                    continue :cond .lt;
 8510                },
 8511                .neq => .ne,
 8512            })));
 8513            const cset_label = isel.instructions.items.len;
 8514            try isel.emit(.subs(.wzr, .w0, .{ .immediate = 0 }));
 8515            try call.finishReturn(isel);
 8516
 8517            try call.prepareCallee(isel);
 8518            try isel.global_relocs.append(isel.pt.zcu.gpa, .{
 8519                .name = switch (bits) {
 8520                    else => unreachable,
 8521                    16 => "__cmphf2",
 8522                    32 => "__cmpsf2",
 8523                    64 => "__cmpdf2",
 8524                    80 => "__cmpxf2",
 8525                    128 => "__cmptf2",
 8526                },
 8527                .reloc = .{ .label = @intCast(isel.instructions.items.len) },
 8528            });
 8529            try isel.emit(.bl(0));
 8530            try call.finishCallee(isel);
 8531
 8532            try call.prepareParams(isel);
 8533            switch (bits) {
 8534                else => unreachable,
 8535                16, 32, 64, 128 => {
 8536                    try call.paramLiveOut(isel, rhs_vi, .v1);
 8537                    try call.paramLiveOut(isel, lhs_vi, .v0);
 8538                },
 8539                80 => {
 8540                    var rhs_hi16_it = rhs_vi.field(ty, 8, 8);
 8541                    const rhs_hi16_vi = try rhs_hi16_it.only(isel);
 8542                    try call.paramLiveOut(isel, rhs_hi16_vi.?, .r3);
 8543                    var rhs_lo64_it = rhs_vi.field(ty, 0, 8);
 8544                    const rhs_lo64_vi = try rhs_lo64_it.only(isel);
 8545                    try call.paramLiveOut(isel, rhs_lo64_vi.?, .r2);
 8546                    var lhs_hi16_it = lhs_vi.field(ty, 8, 8);
 8547                    const lhs_hi16_vi = try lhs_hi16_it.only(isel);
 8548                    try call.paramLiveOut(isel, lhs_hi16_vi.?, .r1);
 8549                    var lhs_lo64_it = lhs_vi.field(ty, 0, 8);
 8550                    const lhs_lo64_vi = try lhs_lo64_it.only(isel);
 8551                    try call.paramLiveOut(isel, lhs_lo64_vi.?, .r0);
 8552                },
 8553            }
 8554            try call.finishParams(isel);
 8555            return .{ .cset_label = cset_label };
 8556        },
 8557    }
 8558}
 8559
 8560fn loadReg(
 8561    isel: *Select,
 8562    ra: Register.Alias,
 8563    size: u64,
 8564    signedness: std.builtin.Signedness,
 8565    base_ra: Register.Alias,
 8566    offset: i65,
 8567) !void {
 8568    switch (size) {
 8569        0 => unreachable,
 8570        1 => {
 8571            if (std.math.cast(u12, offset)) |unsigned_offset| return isel.emit(if (ra.isVector()) .ldr(
 8572                ra.b(),
 8573                .{ .unsigned_offset = .{
 8574                    .base = base_ra.x(),
 8575                    .offset = unsigned_offset,
 8576                } },
 8577            ) else switch (signedness) {
 8578                .signed => .ldrsb(ra.w(), .{ .unsigned_offset = .{
 8579                    .base = base_ra.x(),
 8580                    .offset = unsigned_offset,
 8581                } }),
 8582                .unsigned => .ldrb(ra.w(), .{ .unsigned_offset = .{
 8583                    .base = base_ra.x(),
 8584                    .offset = unsigned_offset,
 8585                } }),
 8586            });
 8587            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
 8588                .ldur(ra.b(), base_ra.x(), signed_offset)
 8589            else switch (signedness) {
 8590                .signed => .ldursb(ra.w(), base_ra.x(), signed_offset),
 8591                .unsigned => .ldurb(ra.w(), base_ra.x(), signed_offset),
 8592            });
 8593        },
 8594        2 => {
 8595            if (std.math.cast(u13, offset)) |unsigned_offset| if (unsigned_offset % 2 == 0)
 8596                return isel.emit(if (ra.isVector()) .ldr(
 8597                    ra.h(),
 8598                    .{ .unsigned_offset = .{
 8599                        .base = base_ra.x(),
 8600                        .offset = unsigned_offset,
 8601                    } },
 8602                ) else switch (signedness) {
 8603                    .signed => .ldrsh(
 8604                        ra.w(),
 8605                        .{ .unsigned_offset = .{
 8606                            .base = base_ra.x(),
 8607                            .offset = unsigned_offset,
 8608                        } },
 8609                    ),
 8610                    .unsigned => .ldrh(
 8611                        ra.w(),
 8612                        .{ .unsigned_offset = .{
 8613                            .base = base_ra.x(),
 8614                            .offset = unsigned_offset,
 8615                        } },
 8616                    ),
 8617                });
 8618            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
 8619                .ldur(ra.h(), base_ra.x(), signed_offset)
 8620            else switch (signedness) {
 8621                .signed => .ldursh(ra.w(), base_ra.x(), signed_offset),
 8622                .unsigned => .ldurh(ra.w(), base_ra.x(), signed_offset),
 8623            });
 8624        },
 8625        3 => {
 8626            const lo16_ra = try isel.allocIntReg();
 8627            defer isel.freeReg(lo16_ra);
 8628            try isel.emit(.orr(ra.w(), lo16_ra.w(), .{ .shifted_register = .{
 8629                .register = ra.w(),
 8630                .shift = .{ .lsl = 16 },
 8631            } }));
 8632            try isel.loadReg(ra, 1, signedness, base_ra, offset + 2);
 8633            return isel.loadReg(lo16_ra, 2, .unsigned, base_ra, offset);
 8634        },
 8635        4 => {
 8636            if (std.math.cast(u14, offset)) |unsigned_offset| if (unsigned_offset % 4 == 0) return isel.emit(.ldr(
 8637                if (ra.isVector()) ra.s() else ra.w(),
 8638                .{ .unsigned_offset = .{
 8639                    .base = base_ra.x(),
 8640                    .offset = unsigned_offset,
 8641                } },
 8642            ));
 8643            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(
 8644                if (ra.isVector()) ra.s() else ra.w(),
 8645                base_ra.x(),
 8646                signed_offset,
 8647            ));
 8648        },
 8649        5, 6 => {
 8650            const lo32_ra = try isel.allocIntReg();
 8651            defer isel.freeReg(lo32_ra);
 8652            try isel.emit(.orr(ra.x(), lo32_ra.x(), .{ .shifted_register = .{
 8653                .register = ra.x(),
 8654                .shift = .{ .lsl = 32 },
 8655            } }));
 8656            try isel.loadReg(ra, size - 4, signedness, base_ra, offset + 4);
 8657            return isel.loadReg(lo32_ra, 4, .unsigned, base_ra, offset);
 8658        },
 8659        7 => {
 8660            const lo32_ra = try isel.allocIntReg();
 8661            defer isel.freeReg(lo32_ra);
 8662            const lo48_ra = try isel.allocIntReg();
 8663            defer isel.freeReg(lo48_ra);
 8664            try isel.emit(.orr(ra.x(), lo48_ra.x(), .{ .shifted_register = .{
 8665                .register = ra.x(),
 8666                .shift = .{ .lsl = 32 + 16 },
 8667            } }));
 8668            try isel.loadReg(ra, 1, signedness, base_ra, offset + 4 + 2);
 8669            try isel.emit(.orr(lo48_ra.x(), lo32_ra.x(), .{ .shifted_register = .{
 8670                .register = lo48_ra.x(),
 8671                .shift = .{ .lsl = 32 },
 8672            } }));
 8673            try isel.loadReg(lo48_ra, 2, .unsigned, base_ra, offset + 4);
 8674            return isel.loadReg(lo32_ra, 4, .unsigned, base_ra, offset);
 8675        },
 8676        8 => {
 8677            if (std.math.cast(u15, offset)) |unsigned_offset| if (unsigned_offset % 8 == 0) return isel.emit(.ldr(
 8678                if (ra.isVector()) ra.d() else ra.x(),
 8679                .{ .unsigned_offset = .{
 8680                    .base = base_ra.x(),
 8681                    .offset = unsigned_offset,
 8682                } },
 8683            ));
 8684            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(
 8685                if (ra.isVector()) ra.d() else ra.x(),
 8686                base_ra.x(),
 8687                signed_offset,
 8688            ));
 8689        },
 8690        16 => {
 8691            if (std.math.cast(u16, offset)) |unsigned_offset| if (unsigned_offset % 16 == 0) return isel.emit(.ldr(
 8692                ra.q(),
 8693                .{ .unsigned_offset = .{
 8694                    .base = base_ra.x(),
 8695                    .offset = unsigned_offset,
 8696                } },
 8697            ));
 8698            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.ldur(ra.q(), base_ra.x(), signed_offset));
 8699        },
 8700        else => return isel.fail("bad load size: {d}", .{size}),
 8701    }
 8702    const ptr_ra = try isel.allocIntReg();
 8703    defer isel.freeReg(ptr_ra);
 8704    try isel.loadReg(ra, size, signedness, ptr_ra, 0);
 8705    if (std.math.cast(u24, offset)) |pos_offset| {
 8706        const lo12: u12 = @truncate(pos_offset >> 0);
 8707        const hi12: u12 = @intCast(pos_offset >> 12);
 8708        if (hi12 > 0) try isel.emit(.add(
 8709            ptr_ra.x(),
 8710            if (lo12 > 0) ptr_ra.x() else base_ra.x(),
 8711            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 8712        ));
 8713        if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
 8714    } else if (std.math.cast(u24, -offset)) |neg_offset| {
 8715        const lo12: u12 = @truncate(neg_offset >> 0);
 8716        const hi12: u12 = @intCast(neg_offset >> 12);
 8717        if (hi12 > 0) try isel.emit(.sub(
 8718            ptr_ra.x(),
 8719            if (lo12 > 0) ptr_ra.x() else base_ra.x(),
 8720            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 8721        ));
 8722        if (lo12 > 0 or hi12 == 0) try isel.emit(.sub(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
 8723    } else {
 8724        try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .register = ptr_ra.x() }));
 8725        try isel.movImmediate(ptr_ra.x(), @truncate(@as(u65, @bitCast(offset))));
 8726    }
 8727}
 8728
 8729fn storeReg(
 8730    isel: *Select,
 8731    ra: Register.Alias,
 8732    size: u64,
 8733    base_ra: Register.Alias,
 8734    offset: i65,
 8735) !void {
 8736    switch (size) {
 8737        0 => unreachable,
 8738        1 => {
 8739            if (std.math.cast(u12, offset)) |unsigned_offset| return isel.emit(if (ra.isVector()) .str(
 8740                ra.b(),
 8741                .{ .unsigned_offset = .{
 8742                    .base = base_ra.x(),
 8743                    .offset = unsigned_offset,
 8744                } },
 8745            ) else .strb(
 8746                ra.w(),
 8747                .{ .unsigned_offset = .{
 8748                    .base = base_ra.x(),
 8749                    .offset = unsigned_offset,
 8750                } },
 8751            ));
 8752            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
 8753                .stur(ra.b(), base_ra.x(), signed_offset)
 8754            else
 8755                .sturb(ra.w(), base_ra.x(), signed_offset));
 8756        },
 8757        2 => {
 8758            if (std.math.cast(u13, offset)) |unsigned_offset| if (unsigned_offset % 2 == 0)
 8759                return isel.emit(if (ra.isVector()) .str(
 8760                    ra.h(),
 8761                    .{ .unsigned_offset = .{
 8762                        .base = base_ra.x(),
 8763                        .offset = unsigned_offset,
 8764                    } },
 8765                ) else .strh(
 8766                    ra.w(),
 8767                    .{ .unsigned_offset = .{
 8768                        .base = base_ra.x(),
 8769                        .offset = unsigned_offset,
 8770                    } },
 8771                ));
 8772            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(if (ra.isVector())
 8773                .stur(ra.h(), base_ra.x(), signed_offset)
 8774            else
 8775                .sturh(ra.w(), base_ra.x(), signed_offset));
 8776        },
 8777        3 => {
 8778            const hi8_ra = try isel.allocIntReg();
 8779            defer isel.freeReg(hi8_ra);
 8780            try isel.storeReg(hi8_ra, 1, base_ra, offset + 2);
 8781            try isel.storeReg(ra, 2, base_ra, offset);
 8782            return isel.emit(.ubfm(hi8_ra.w(), ra.w(), .{
 8783                .N = .word,
 8784                .immr = 16,
 8785                .imms = 16 + 8 - 1,
 8786            }));
 8787        },
 8788        4 => {
 8789            if (std.math.cast(u14, offset)) |unsigned_offset| if (unsigned_offset % 4 == 0) return isel.emit(.str(
 8790                if (ra.isVector()) ra.s() else ra.w(),
 8791                .{ .unsigned_offset = .{
 8792                    .base = base_ra.x(),
 8793                    .offset = unsigned_offset,
 8794                } },
 8795            ));
 8796            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(
 8797                if (ra.isVector()) ra.s() else ra.w(),
 8798                base_ra.x(),
 8799                signed_offset,
 8800            ));
 8801        },
 8802        5 => {
 8803            const hi8_ra = try isel.allocIntReg();
 8804            defer isel.freeReg(hi8_ra);
 8805            try isel.storeReg(hi8_ra, 1, base_ra, offset + 4);
 8806            try isel.storeReg(ra, 4, base_ra, offset);
 8807            return isel.emit(.ubfm(hi8_ra.x(), ra.x(), .{
 8808                .N = .doubleword,
 8809                .immr = 32,
 8810                .imms = 32 + 8 - 1,
 8811            }));
 8812        },
 8813        6 => {
 8814            const hi16_ra = try isel.allocIntReg();
 8815            defer isel.freeReg(hi16_ra);
 8816            try isel.storeReg(hi16_ra, 2, base_ra, offset + 4);
 8817            try isel.storeReg(ra, 4, base_ra, offset);
 8818            return isel.emit(.ubfm(hi16_ra.x(), ra.x(), .{
 8819                .N = .doubleword,
 8820                .immr = 32,
 8821                .imms = 32 + 16 - 1,
 8822            }));
 8823        },
 8824        7 => {
 8825            const hi16_ra = try isel.allocIntReg();
 8826            defer isel.freeReg(hi16_ra);
 8827            const hi8_ra = try isel.allocIntReg();
 8828            defer isel.freeReg(hi8_ra);
 8829            try isel.storeReg(hi8_ra, 1, base_ra, offset + 6);
 8830            try isel.storeReg(hi16_ra, 2, base_ra, offset + 4);
 8831            try isel.storeReg(ra, 4, base_ra, offset);
 8832            try isel.emit(.ubfm(hi8_ra.x(), ra.x(), .{
 8833                .N = .doubleword,
 8834                .immr = 32 + 16,
 8835                .imms = 32 + 16 + 8 - 1,
 8836            }));
 8837            return isel.emit(.ubfm(hi16_ra.x(), ra.x(), .{
 8838                .N = .doubleword,
 8839                .immr = 32,
 8840                .imms = 32 + 16 - 1,
 8841            }));
 8842        },
 8843        8 => {
 8844            if (std.math.cast(u15, offset)) |unsigned_offset| if (unsigned_offset % 8 == 0) return isel.emit(.str(
 8845                if (ra.isVector()) ra.d() else ra.x(),
 8846                .{ .unsigned_offset = .{
 8847                    .base = base_ra.x(),
 8848                    .offset = unsigned_offset,
 8849                } },
 8850            ));
 8851            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(
 8852                if (ra.isVector()) ra.d() else ra.x(),
 8853                base_ra.x(),
 8854                signed_offset,
 8855            ));
 8856        },
 8857        16 => {
 8858            if (std.math.cast(u16, offset)) |unsigned_offset| if (unsigned_offset % 16 == 0) return isel.emit(.str(
 8859                ra.q(),
 8860                .{ .unsigned_offset = .{
 8861                    .base = base_ra.x(),
 8862                    .offset = unsigned_offset,
 8863                } },
 8864            ));
 8865            if (std.math.cast(i9, offset)) |signed_offset| return isel.emit(.stur(ra.q(), base_ra.x(), signed_offset));
 8866        },
 8867        else => return isel.fail("bad store size: {d}", .{size}),
 8868    }
 8869    const ptr_ra = try isel.allocIntReg();
 8870    defer isel.freeReg(ptr_ra);
 8871    try isel.storeReg(ra, size, ptr_ra, 0);
 8872    if (std.math.cast(u24, offset)) |pos_offset| {
 8873        const lo12: u12 = @truncate(pos_offset >> 0);
 8874        const hi12: u12 = @intCast(pos_offset >> 12);
 8875        if (hi12 > 0) try isel.emit(.add(
 8876            ptr_ra.x(),
 8877            if (lo12 > 0) ptr_ra.x() else base_ra.x(),
 8878            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 8879        ));
 8880        if (lo12 > 0 or hi12 == 0) try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
 8881    } else if (std.math.cast(u24, -offset)) |neg_offset| {
 8882        const lo12: u12 = @truncate(neg_offset >> 0);
 8883        const hi12: u12 = @intCast(neg_offset >> 12);
 8884        if (hi12 > 0) try isel.emit(.sub(
 8885            ptr_ra.x(),
 8886            if (lo12 > 0) ptr_ra.x() else base_ra.x(),
 8887            .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
 8888        ));
 8889        if (lo12 > 0 or hi12 == 0) try isel.emit(.sub(ptr_ra.x(), base_ra.x(), .{ .immediate = lo12 }));
 8890    } else {
 8891        try isel.emit(.add(ptr_ra.x(), base_ra.x(), .{ .register = ptr_ra.x() }));
 8892        try isel.movImmediate(ptr_ra.x(), @truncate(@as(u65, @bitCast(offset))));
 8893    }
 8894}
 8895
 8896const DomInt = u8;
 8897
 8898pub const Value = struct {
 8899    refs: u32,
 8900    flags: Flags,
 8901    offset_from_parent: u64,
 8902    parent_payload: Parent.Payload,
 8903    location_payload: Location.Payload,
 8904    parts: Value.Index,
 8905
 8906    /// Must be at least 16 to compute call abi.
 8907    /// Must be at least 16, the largest hardware alignment.
 8908    pub const max_parts = 16;
 8909    pub const PartsLen = std.math.IntFittingRange(0, Value.max_parts);
 8910
 8911    comptime {
 8912        if (!std.debug.runtime_safety) assert(@sizeOf(Value) == 32);
 8913    }
 8914
 8915    pub const Flags = packed struct(u32) {
 8916        alignment: InternPool.Alignment,
 8917        parent_tag: Parent.Tag,
 8918        location_tag: Location.Tag,
 8919        parts_len_minus_one: std.math.IntFittingRange(0, Value.max_parts - 1),
 8920        unused: u18 = 0,
 8921    };
 8922
 8923    pub const Parent = union(enum(u3)) {
 8924        unallocated: void,
 8925        stack_slot: Indirect,
 8926        address: Value.Index,
 8927        value: Value.Index,
 8928        constant: Constant,
 8929
 8930        pub const Tag = @typeInfo(Parent).@"union".tag_type.?;
 8931        pub const Payload = Payload: {
 8932            const fields = @typeInfo(Parent).@"union".fields;
 8933            var types: [fields.len]type = undefined;
 8934            var names: [fields.len][]const u8 = undefined;
 8935            for (fields, &types, &names) |f, *ty, *name| {
 8936                ty.* = f.type;
 8937                name.* = f.name;
 8938            }
 8939            break :Payload @Union(.auto, null, &names, &types, &@splat(.{}));
 8940        };
 8941    };
 8942
 8943    pub const Location = union(enum(u1)) {
 8944        large: struct {
 8945            size: u64,
 8946        },
 8947        small: struct {
 8948            size: u5,
 8949            signedness: std.builtin.Signedness,
 8950            is_vector: bool,
 8951            hint: Register.Alias,
 8952            register: Register.Alias,
 8953        },
 8954
 8955        pub const Tag = @typeInfo(Location).@"union".tag_type.?;
 8956        pub const Payload = Payload: {
 8957            const fields = @typeInfo(Location).@"union".fields;
 8958            var types: [fields.len]type = undefined;
 8959            var names: [fields.len][]const u8 = undefined;
 8960            for (fields, &types, &names) |f, *ty, *name| {
 8961                ty.* = f.type;
 8962                name.* = f.name;
 8963            }
 8964            break :Payload @Union(.auto, null, &names, &types, &@splat(.{}));
 8965        };
 8966    };
 8967
 8968    pub const Indirect = packed struct(u32) {
 8969        base: Register.Alias,
 8970        offset: i25,
 8971
 8972        pub fn withOffset(ind: Indirect, offset: i25) Indirect {
 8973            return .{
 8974                .base = ind.base,
 8975                .offset = ind.offset + offset,
 8976            };
 8977        }
 8978    };
 8979
 8980    pub const Index = enum(u32) {
 8981        allocating = std.math.maxInt(u32) - 1,
 8982        free = std.math.maxInt(u32) - 0,
 8983        _,
 8984
 8985        fn get(vi: Value.Index, isel: *Select) *Value {
 8986            return &isel.values.items[@intFromEnum(vi)];
 8987        }
 8988
 8989        fn setAlignment(vi: Value.Index, isel: *Select, new_alignment: InternPool.Alignment) void {
 8990            vi.get(isel).flags.alignment = new_alignment;
 8991        }
 8992
 8993        pub fn alignment(vi: Value.Index, isel: *Select) InternPool.Alignment {
 8994            return vi.get(isel).flags.alignment;
 8995        }
 8996
 8997        pub fn setParent(vi: Value.Index, isel: *Select, new_parent: Parent) void {
 8998            const value = vi.get(isel);
 8999            assert(value.flags.parent_tag == .unallocated);
 9000            value.flags.parent_tag = new_parent;
 9001            value.parent_payload = switch (new_parent) {
 9002                .unallocated => unreachable,
 9003                inline else => |payload, tag| @unionInit(Parent.Payload, @tagName(tag), payload),
 9004            };
 9005            if (value.refs > 0) switch (new_parent) {
 9006                .unallocated => unreachable,
 9007                .stack_slot, .constant => {},
 9008                .address, .value => |parent_vi| _ = parent_vi.ref(isel),
 9009            };
 9010        }
 9011
 9012        pub fn changeStackSlot(vi: Value.Index, isel: *Select, new_stack_slot: Indirect) void {
 9013            const value = vi.get(isel);
 9014            assert(value.flags.parent_tag == .stack_slot);
 9015            value.flags.parent_tag = .unallocated;
 9016            vi.setParent(isel, .{ .stack_slot = new_stack_slot });
 9017        }
 9018
 9019        pub fn parent(vi: Value.Index, isel: *Select) Parent {
 9020            const value = vi.get(isel);
 9021            return switch (value.flags.parent_tag) {
 9022                inline else => |tag| @unionInit(
 9023                    Parent,
 9024                    @tagName(tag),
 9025                    @field(value.parent_payload, @tagName(tag)),
 9026                ),
 9027            };
 9028        }
 9029
 9030        pub fn valueParent(initial_vi: Value.Index, isel: *Select) struct { u64, Value.Index } {
 9031            var offset: u64 = 0;
 9032            var vi = initial_vi;
 9033            parent: switch (vi.parent(isel)) {
 9034                else => return .{ offset, vi },
 9035                .value => |parent_vi| {
 9036                    offset += vi.position(isel)[0];
 9037                    vi = parent_vi;
 9038                    continue :parent parent_vi.parent(isel);
 9039                },
 9040            }
 9041        }
 9042
 9043        pub fn location(vi: Value.Index, isel: *Select) Location {
 9044            const value = vi.get(isel);
 9045            return switch (value.flags.location_tag) {
 9046                inline else => |tag| @unionInit(
 9047                    Location,
 9048                    @tagName(tag),
 9049                    @field(value.location_payload, @tagName(tag)),
 9050                ),
 9051            };
 9052        }
 9053
 9054        pub fn position(vi: Value.Index, isel: *Select) struct { u64, u64 } {
 9055            return .{ vi.get(isel).offset_from_parent, vi.size(isel) };
 9056        }
 9057
 9058        pub fn size(vi: Value.Index, isel: *Select) u64 {
 9059            return switch (vi.location(isel)) {
 9060                inline else => |loc| loc.size,
 9061            };
 9062        }
 9063
 9064        fn setHint(vi: Value.Index, isel: *Select, new_hint: Register.Alias) void {
 9065            vi.get(isel).location_payload.small.hint = new_hint;
 9066        }
 9067
 9068        pub fn hint(vi: Value.Index, isel: *Select) ?Register.Alias {
 9069            return switch (vi.location(isel)) {
 9070                .large => null,
 9071                .small => |loc| switch (loc.hint) {
 9072                    .zr => null,
 9073                    else => |hint_reg| hint_reg,
 9074                },
 9075            };
 9076        }
 9077
 9078        fn setSignedness(vi: Value.Index, isel: *Select, new_signedness: std.builtin.Signedness) void {
 9079            const value = vi.get(isel);
 9080            assert(value.location_payload.small.size <= 2);
 9081            value.location_payload.small.signedness = new_signedness;
 9082        }
 9083
 9084        pub fn signedness(vi: Value.Index, isel: *Select) std.builtin.Signedness {
 9085            const value = vi.get(isel);
 9086            return switch (value.flags.location_tag) {
 9087                .large => .unsigned,
 9088                .small => value.location_payload.small.signedness,
 9089            };
 9090        }
 9091
 9092        fn setIsVector(vi: Value.Index, isel: *Select) void {
 9093            const is_vector = &vi.get(isel).location_payload.small.is_vector;
 9094            assert(!is_vector.*);
 9095            is_vector.* = true;
 9096        }
 9097
 9098        pub fn isVector(vi: Value.Index, isel: *Select) bool {
 9099            const value = vi.get(isel);
 9100            return switch (value.flags.location_tag) {
 9101                .large => false,
 9102                .small => value.location_payload.small.is_vector,
 9103            };
 9104        }
 9105
 9106        pub fn register(vi: Value.Index, isel: *Select) ?Register.Alias {
 9107            return switch (vi.location(isel)) {
 9108                .large => null,
 9109                .small => |loc| switch (loc.register) {
 9110                    .zr => null,
 9111                    else => |reg| reg,
 9112                },
 9113            };
 9114        }
 9115
 9116        pub fn isUsed(vi: Value.Index, isel: *Select) bool {
 9117            return vi.valueParent(isel)[1].parent(isel) != .unallocated or vi.hasRegisterRecursive(isel);
 9118        }
 9119
 9120        fn hasRegisterRecursive(vi: Value.Index, isel: *Select) bool {
 9121            if (vi.register(isel)) |_| return true;
 9122            var part_it = vi.parts(isel);
 9123            if (part_it.only() == null) while (part_it.next()) |part_vi| if (part_vi.hasRegisterRecursive(isel)) return true;
 9124            return false;
 9125        }
 9126
 9127        fn setParts(vi: Value.Index, isel: *Select, parts_len: Value.PartsLen) void {
 9128            assert(parts_len > 1);
 9129            const value = vi.get(isel);
 9130            assert(value.flags.parts_len_minus_one == 0);
 9131            value.parts = @enumFromInt(isel.values.items.len);
 9132            value.flags.parts_len_minus_one = @intCast(parts_len - 1);
 9133        }
 9134
 9135        fn addPart(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64) Value.Index {
 9136            const part_vi = isel.initValueAdvanced(vi.alignment(isel), part_offset, part_size);
 9137            tracking_log.debug("${d} <- ${d}[{d}]", .{
 9138                @intFromEnum(part_vi),
 9139                @intFromEnum(vi),
 9140                part_offset,
 9141            });
 9142            part_vi.setParent(isel, .{ .value = vi });
 9143            return part_vi;
 9144        }
 9145
 9146        pub fn parts(vi: Value.Index, isel: *Select) Value.PartIterator {
 9147            const value = vi.get(isel);
 9148            return switch (value.flags.parts_len_minus_one) {
 9149                0 => .initOne(vi),
 9150                else => |parts_len_minus_one| .{
 9151                    .vi = value.parts,
 9152                    .remaining = @as(Value.PartsLen, parts_len_minus_one) + 1,
 9153                },
 9154            };
 9155        }
 9156
 9157        fn containingParts(vi: Value.Index, isel: *Select, part_offset: u64, part_size: u64) Value.PartIterator {
 9158            const start_vi = vi.partAtOffset(isel, part_offset);
 9159            const start_offset, const start_size = start_vi.position(isel);
 9160            if (part_offset >= start_offset and part_size <= start_size) return .initOne(start_vi);
 9161            const end_vi = vi.partAtOffset(isel, part_size - 1 + part_offset);
 9162            return .{
 9163                .vi = start_vi,
 9164                .remaining = @intCast(@intFromEnum(end_vi) - @intFromEnum(start_vi) + 1),
 9165            };
 9166        }
 9167        comptime {
 9168            _ = containingParts;
 9169        }
 9170
 9171        fn partAtOffset(vi: Value.Index, isel: *Select, offset: u64) Value.Index {
 9172            const SearchPartIndex = std.math.IntFittingRange(0, Value.max_parts * 2 - 1);
 9173            const value = vi.get(isel);
 9174            var last: SearchPartIndex = value.flags.parts_len_minus_one;
 9175            if (last == 0) return vi;
 9176            var first: SearchPartIndex = 0;
 9177            last += 1;
 9178            while (true) {
 9179                const mid = (first + last) / 2;
 9180                const mid_vi: Value.Index = @enumFromInt(@intFromEnum(value.parts) + mid);
 9181                if (mid == first) return mid_vi;
 9182                if (offset < mid_vi.get(isel).offset_from_parent) last = mid else first = mid;
 9183            }
 9184        }
 9185
 9186        fn field(
 9187            vi: Value.Index,
 9188            ty: ZigType,
 9189            field_offset: u64,
 9190            field_size: u64,
 9191        ) Value.FieldPartIterator {
 9192            assert(field_size > 0);
 9193            return .{
 9194                .vi = vi,
 9195                .ty = ty,
 9196                .field_offset = field_offset,
 9197                .field_size = field_size,
 9198                .next_offset = 0,
 9199            };
 9200        }
 9201
 9202        fn ref(initial_vi: Value.Index, isel: *Select) Value.Index {
 9203            var vi = initial_vi;
 9204            while (true) {
 9205                const refs = &vi.get(isel).refs;
 9206                refs.* += 1;
 9207                if (refs.* > 1) return initial_vi;
 9208                switch (vi.parent(isel)) {
 9209                    .unallocated, .stack_slot, .constant => {},
 9210                    .address, .value => |parent_vi| {
 9211                        vi = parent_vi;
 9212                        continue;
 9213                    },
 9214                }
 9215                return initial_vi;
 9216            }
 9217        }
 9218
 9219        pub fn deref(initial_vi: Value.Index, isel: *Select) void {
 9220            var vi = initial_vi;
 9221            while (true) {
 9222                const refs = &vi.get(isel).refs;
 9223                refs.* -= 1;
 9224                if (refs.* > 0) return;
 9225                switch (vi.parent(isel)) {
 9226                    .unallocated, .constant => {},
 9227                    .stack_slot => {
 9228                        // reuse stack slot
 9229                    },
 9230                    .address, .value => |parent_vi| {
 9231                        vi = parent_vi;
 9232                        continue;
 9233                    },
 9234                }
 9235                return;
 9236            }
 9237        }
 9238
 9239        fn move(dst_vi: Value.Index, isel: *Select, src_ref: Air.Inst.Ref) !void {
 9240            try dst_vi.copy(
 9241                isel,
 9242                isel.air.typeOf(src_ref, &isel.pt.zcu.intern_pool),
 9243                try isel.use(src_ref),
 9244            );
 9245        }
 9246
 9247        fn copy(dst_vi: Value.Index, isel: *Select, ty: ZigType, src_vi: Value.Index) !void {
 9248            try dst_vi.copyAdvanced(isel, src_vi, .{
 9249                .ty = ty,
 9250                .dst_vi = dst_vi,
 9251                .dst_offset = 0,
 9252                .src_vi = src_vi,
 9253                .src_offset = 0,
 9254            });
 9255        }
 9256
 9257        fn copyAdvanced(dst_vi: Value.Index, isel: *Select, src_vi: Value.Index, root: struct {
 9258            ty: ZigType,
 9259            dst_vi: Value.Index,
 9260            dst_offset: u64,
 9261            src_vi: Value.Index,
 9262            src_offset: u64,
 9263        }) !void {
 9264            if (dst_vi == src_vi) return;
 9265            var dst_part_it = dst_vi.parts(isel);
 9266            if (dst_part_it.only()) |dst_part_vi| {
 9267                var src_part_it = src_vi.parts(isel);
 9268                if (src_part_it.only()) |src_part_vi| only: {
 9269                    const src_part_size = src_part_vi.size(isel);
 9270                    if (src_part_size > @as(@TypeOf(src_part_size), if (src_part_vi.isVector(isel)) 16 else 8)) {
 9271                        var subpart_it = root.src_vi.field(root.ty, root.src_offset, src_part_size - 1);
 9272                        _ = try subpart_it.next(isel);
 9273                        src_part_it = src_vi.parts(isel);
 9274                        assert(src_part_it.only() == null);
 9275                        break :only;
 9276                    }
 9277                    return src_part_vi.liveOut(isel, try dst_part_vi.defReg(isel) orelse return);
 9278                }
 9279                while (src_part_it.next()) |src_part_vi| {
 9280                    const src_part_offset, const src_part_size = src_part_vi.position(isel);
 9281                    var dst_field_it = root.dst_vi.field(root.ty, root.dst_offset + src_part_offset, src_part_size);
 9282                    const dst_field_vi = try dst_field_it.only(isel);
 9283                    try dst_field_vi.?.copyAdvanced(isel, src_part_vi, .{
 9284                        .ty = root.ty,
 9285                        .dst_vi = root.dst_vi,
 9286                        .dst_offset = root.dst_offset + src_part_offset,
 9287                        .src_vi = root.src_vi,
 9288                        .src_offset = root.src_offset + src_part_offset,
 9289                    });
 9290                }
 9291            } else while (dst_part_it.next()) |dst_part_vi| {
 9292                const dst_part_offset, const dst_part_size = dst_part_vi.position(isel);
 9293                var src_field_it = root.src_vi.field(root.ty, root.src_offset + dst_part_offset, dst_part_size);
 9294                const src_part_vi = try src_field_it.only(isel);
 9295                try dst_part_vi.copyAdvanced(isel, src_part_vi.?, .{
 9296                    .ty = root.ty,
 9297                    .dst_vi = root.dst_vi,
 9298                    .dst_offset = root.dst_offset + dst_part_offset,
 9299                    .src_vi = root.src_vi,
 9300                    .src_offset = root.src_offset + dst_part_offset,
 9301                });
 9302            }
 9303        }
 9304
 9305        const AddOrSubtractOptions = struct {
 9306            overflow: Overflow,
 9307
 9308            const Overflow = union(enum) {
 9309                @"unreachable",
 9310                panic: Zcu.SimplePanicId,
 9311                wrap,
 9312                ra: Register.Alias,
 9313
 9314                fn defCond(overflow: Overflow, isel: *Select, cond: codegen.aarch64.encoding.ConditionCode) !void {
 9315                    switch (overflow) {
 9316                        .@"unreachable" => unreachable,
 9317                        .panic => |panic_id| {
 9318                            const skip_label = isel.instructions.items.len;
 9319                            try isel.emitPanic(panic_id);
 9320                            try isel.emit(.@"b."(
 9321                                cond.invert(),
 9322                                @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 9323                            ));
 9324                        },
 9325                        .wrap => {},
 9326                        .ra => |overflow_ra| try isel.emit(.csinc(overflow_ra.w(), .wzr, .wzr, cond.invert())),
 9327                    }
 9328                }
 9329            };
 9330        };
 9331        fn addOrSubtract(
 9332            res_vi: Value.Index,
 9333            isel: *Select,
 9334            ty: ZigType,
 9335            lhs_vi: Value.Index,
 9336            op: codegen.aarch64.encoding.Instruction.AddSubtractOp,
 9337            rhs_vi: Value.Index,
 9338            opts: AddOrSubtractOptions,
 9339        ) !void {
 9340            const zcu = isel.pt.zcu;
 9341            if (!ty.isAbiInt(zcu)) return isel.fail("bad {t} {f}", .{ op, isel.fmtType(ty) });
 9342            const int_info = ty.intInfo(zcu);
 9343            if (int_info.bits > 128) return isel.fail("too big {t} {f}", .{ op, isel.fmtType(ty) });
 9344            var part_offset = res_vi.size(isel);
 9345            var need_wrap = switch (opts.overflow) {
 9346                .@"unreachable" => false,
 9347                .panic, .wrap, .ra => true,
 9348            };
 9349            var need_carry = switch (opts.overflow) {
 9350                .@"unreachable", .wrap => false,
 9351                .panic, .ra => true,
 9352            };
 9353            while (part_offset > 0) : (need_wrap = false) {
 9354                const part_size = @min(part_offset, 8);
 9355                part_offset -= part_size;
 9356                var wrapped_res_part_it = res_vi.field(ty, part_offset, part_size);
 9357                const wrapped_res_part_vi = try wrapped_res_part_it.only(isel);
 9358                const wrapped_res_part_ra = wrapped_res_part_ra: {
 9359                    const overflow_ra_lock: RegLock = switch (opts.overflow) {
 9360                        .ra => |ra| isel.lockReg(ra),
 9361                        else => .empty,
 9362                    };
 9363                    defer overflow_ra_lock.unlock(isel);
 9364                    break :wrapped_res_part_ra try wrapped_res_part_vi.?.defReg(isel) orelse if (need_carry) .zr else continue;
 9365                };
 9366                const unwrapped_res_part_ra = unwrapped_res_part_ra: {
 9367                    if (!need_wrap) break :unwrapped_res_part_ra wrapped_res_part_ra;
 9368                    if (int_info.bits % 32 == 0) {
 9369                        try opts.overflow.defCond(isel, switch (int_info.signedness) {
 9370                            .signed => .vs,
 9371                            .unsigned => switch (op) {
 9372                                .add => .cs,
 9373                                .sub => .cc,
 9374                            },
 9375                        });
 9376                        break :unwrapped_res_part_ra wrapped_res_part_ra;
 9377                    }
 9378                    need_carry = false;
 9379                    const wrapped_part_ra, const unwrapped_part_ra = part_ra: switch (opts.overflow) {
 9380                        .@"unreachable" => unreachable,
 9381                        .panic, .ra => switch (int_info.signedness) {
 9382                            .signed => {
 9383                                try opts.overflow.defCond(isel, .ne);
 9384                                const wrapped_part_ra = switch (wrapped_res_part_ra) {
 9385                                    else => |res_part_ra| res_part_ra,
 9386                                    .zr => try isel.allocIntReg(),
 9387                                };
 9388                                errdefer if (wrapped_part_ra != wrapped_res_part_ra) isel.freeReg(wrapped_part_ra);
 9389                                const unwrapped_part_ra = unwrapped_part_ra: {
 9390                                    const wrapped_res_part_lock: RegLock = switch (wrapped_res_part_ra) {
 9391                                        else => |res_part_ra| isel.lockReg(res_part_ra),
 9392                                        .zr => .empty,
 9393                                    };
 9394                                    defer wrapped_res_part_lock.unlock(isel);
 9395                                    break :unwrapped_part_ra try isel.allocIntReg();
 9396                                };
 9397                                errdefer isel.freeReg(unwrapped_part_ra);
 9398                                switch (part_size) {
 9399                                    else => unreachable,
 9400                                    1...4 => try isel.emit(.subs(.wzr, wrapped_part_ra.w(), .{ .register = unwrapped_part_ra.w() })),
 9401                                    5...8 => try isel.emit(.subs(.xzr, wrapped_part_ra.x(), .{ .register = unwrapped_part_ra.x() })),
 9402                                }
 9403                                break :part_ra .{ wrapped_part_ra, unwrapped_part_ra };
 9404                            },
 9405                            .unsigned => {
 9406                                const unwrapped_part_ra = unwrapped_part_ra: {
 9407                                    const wrapped_res_part_lock: RegLock = switch (wrapped_res_part_ra) {
 9408                                        else => |res_part_ra| isel.lockReg(res_part_ra),
 9409                                        .zr => .empty,
 9410                                    };
 9411                                    defer wrapped_res_part_lock.unlock(isel);
 9412                                    break :unwrapped_part_ra try isel.allocIntReg();
 9413                                };
 9414                                errdefer isel.freeReg(unwrapped_part_ra);
 9415                                const bit: u6 = @truncate(int_info.bits);
 9416                                switch (opts.overflow) {
 9417                                    .@"unreachable", .wrap => unreachable,
 9418                                    .panic => |panic_id| {
 9419                                        const skip_label = isel.instructions.items.len;
 9420                                        try isel.emitPanic(panic_id);
 9421                                        try isel.emit(.tbz(
 9422                                            switch (bit) {
 9423                                                0, 32 => unreachable,
 9424                                                1...31 => unwrapped_part_ra.w(),
 9425                                                33...63 => unwrapped_part_ra.x(),
 9426                                            },
 9427                                            bit,
 9428                                            @intCast((isel.instructions.items.len + 1 - skip_label) << 2),
 9429                                        ));
 9430                                    },
 9431                                    .ra => |overflow_ra| try isel.emit(switch (bit) {
 9432                                        0, 32 => unreachable,
 9433                                        1...31 => .ubfm(overflow_ra.w(), unwrapped_part_ra.w(), .{
 9434                                            .N = .word,
 9435                                            .immr = bit,
 9436                                            .imms = bit,
 9437                                        }),
 9438                                        33...63 => .ubfm(overflow_ra.x(), unwrapped_part_ra.x(), .{
 9439                                            .N = .doubleword,
 9440                                            .immr = bit,
 9441                                            .imms = bit,
 9442                                        }),
 9443                                    }),
 9444                                }
 9445                                break :part_ra .{ wrapped_res_part_ra, unwrapped_part_ra };
 9446                            },
 9447                        },
 9448                        .wrap => .{ wrapped_res_part_ra, wrapped_res_part_ra },
 9449                    };
 9450                    defer if (wrapped_part_ra != wrapped_res_part_ra) isel.freeReg(wrapped_part_ra);
 9451                    errdefer if (unwrapped_part_ra != wrapped_res_part_ra) isel.freeReg(unwrapped_part_ra);
 9452                    if (wrapped_part_ra != .zr) try isel.emit(switch (part_size) {
 9453                        else => unreachable,
 9454                        1...4 => switch (int_info.signedness) {
 9455                            .signed => .sbfm(wrapped_part_ra.w(), unwrapped_part_ra.w(), .{
 9456                                .N = .word,
 9457                                .immr = 0,
 9458                                .imms = @truncate(int_info.bits - 1),
 9459                            }),
 9460                            .unsigned => .ubfm(wrapped_part_ra.w(), unwrapped_part_ra.w(), .{
 9461                                .N = .word,
 9462                                .immr = 0,
 9463                                .imms = @truncate(int_info.bits - 1),
 9464                            }),
 9465                        },
 9466                        5...8 => switch (int_info.signedness) {
 9467                            .signed => .sbfm(wrapped_part_ra.x(), unwrapped_part_ra.x(), .{
 9468                                .N = .doubleword,
 9469                                .immr = 0,
 9470                                .imms = @truncate(int_info.bits - 1),
 9471                            }),
 9472                            .unsigned => .ubfm(wrapped_part_ra.x(), unwrapped_part_ra.x(), .{
 9473                                .N = .doubleword,
 9474                                .immr = 0,
 9475                                .imms = @truncate(int_info.bits - 1),
 9476                            }),
 9477                        },
 9478                    });
 9479                    break :unwrapped_res_part_ra unwrapped_part_ra;
 9480                };
 9481                defer if (unwrapped_res_part_ra != wrapped_res_part_ra) isel.freeReg(unwrapped_res_part_ra);
 9482                var lhs_part_it = lhs_vi.field(ty, part_offset, part_size);
 9483                const lhs_part_vi = try lhs_part_it.only(isel);
 9484                const lhs_part_mat = try lhs_part_vi.?.matReg(isel);
 9485                var rhs_part_it = rhs_vi.field(ty, part_offset, part_size);
 9486                const rhs_part_vi = try rhs_part_it.only(isel);
 9487                const rhs_part_mat = try rhs_part_vi.?.matReg(isel);
 9488                try isel.emit(switch (part_size) {
 9489                    else => unreachable,
 9490                    1...4 => switch (op) {
 9491                        .add => switch (part_offset) {
 9492                            0 => switch (need_carry) {
 9493                                false => .add(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 9494                                true => .adds(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 9495                            },
 9496                            else => switch (need_carry) {
 9497                                false => .adc(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
 9498                                true => .adcs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
 9499                            },
 9500                        },
 9501                        .sub => switch (part_offset) {
 9502                            0 => switch (need_carry) {
 9503                                false => .sub(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 9504                                true => .subs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), .{ .register = rhs_part_mat.ra.w() }),
 9505                            },
 9506                            else => switch (need_carry) {
 9507                                false => .sbc(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
 9508                                true => .sbcs(unwrapped_res_part_ra.w(), lhs_part_mat.ra.w(), rhs_part_mat.ra.w()),
 9509                            },
 9510                        },
 9511                    },
 9512                    5...8 => switch (op) {
 9513                        .add => switch (part_offset) {
 9514                            0 => switch (need_carry) {
 9515                                false => .add(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 9516                                true => .adds(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 9517                            },
 9518                            else => switch (need_carry) {
 9519                                false => .adc(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
 9520                                true => .adcs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
 9521                            },
 9522                        },
 9523                        .sub => switch (part_offset) {
 9524                            0 => switch (need_carry) {
 9525                                false => .sub(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 9526                                true => .subs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), .{ .register = rhs_part_mat.ra.x() }),
 9527                            },
 9528                            else => switch (need_carry) {
 9529                                false => .sbc(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
 9530                                true => .sbcs(unwrapped_res_part_ra.x(), lhs_part_mat.ra.x(), rhs_part_mat.ra.x()),
 9531                            },
 9532                        },
 9533                    },
 9534                });
 9535                try rhs_part_mat.finish(isel);
 9536                try lhs_part_mat.finish(isel);
 9537                need_carry = true;
 9538            }
 9539        }
 9540
 9541        const MemoryAccessOptions = struct {
 9542            root_vi: Value.Index = .free,
 9543            offset: u64 = 0,
 9544            @"volatile": bool = false,
 9545            split: bool = true,
 9546            wrap: ?std.builtin.Type.Int = null,
 9547            expected_live_registers: *const LiveRegisters = &.initFill(.free),
 9548        };
 9549
 9550        fn load(
 9551            vi: Value.Index,
 9552            isel: *Select,
 9553            root_ty: ZigType,
 9554            base_ra: Register.Alias,
 9555            opts: MemoryAccessOptions,
 9556        ) !bool {
 9557            const root_vi = switch (opts.root_vi) {
 9558                _ => |root_vi| root_vi,
 9559                .allocating => unreachable,
 9560                .free => vi,
 9561            };
 9562            var part_it = vi.parts(isel);
 9563            if (part_it.only()) |part_vi| only: {
 9564                const part_size = part_vi.size(isel);
 9565                const part_is_vector = part_vi.isVector(isel);
 9566                if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
 9567                    if (!opts.split) return false;
 9568                    var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
 9569                    _ = try subpart_it.next(isel);
 9570                    part_it = vi.parts(isel);
 9571                    assert(part_it.only() == null);
 9572                    break :only;
 9573                }
 9574                const part_ra = if (try part_vi.defReg(isel)) |part_ra|
 9575                    part_ra
 9576                else if (opts.@"volatile")
 9577                    .zr
 9578                else
 9579                    return false;
 9580                const part_lock: RegLock = switch (part_ra) {
 9581                    else => isel.lockReg(part_ra),
 9582                    .zr => .empty,
 9583                };
 9584                defer switch (opts.expected_live_registers.get(part_ra)) {
 9585                    _ => {},
 9586                    .allocating => unreachable,
 9587                    .free => part_lock.unlock(isel),
 9588                };
 9589                if (opts.wrap) |int_info| switch (int_info.bits) {
 9590                    else => unreachable,
 9591                    1...7, 9...15, 17...31 => |bits| try isel.emit(switch (int_info.signedness) {
 9592                        .signed => .sbfm(part_ra.w(), part_ra.w(), .{
 9593                            .N = .word,
 9594                            .immr = 0,
 9595                            .imms = @intCast(bits - 1),
 9596                        }),
 9597                        .unsigned => .ubfm(part_ra.w(), part_ra.w(), .{
 9598                            .N = .word,
 9599                            .immr = 0,
 9600                            .imms = @intCast(bits - 1),
 9601                        }),
 9602                    }),
 9603                    8, 16, 32 => {},
 9604                    33...63 => |bits| try isel.emit(switch (int_info.signedness) {
 9605                        .signed => .sbfm(part_ra.x(), part_ra.x(), .{
 9606                            .N = .doubleword,
 9607                            .immr = 0,
 9608                            .imms = @intCast(bits - 1),
 9609                        }),
 9610                        .unsigned => .ubfm(part_ra.x(), part_ra.x(), .{
 9611                            .N = .doubleword,
 9612                            .immr = 0,
 9613                            .imms = @intCast(bits - 1),
 9614                        }),
 9615                    }),
 9616                    64 => {},
 9617                };
 9618                try isel.loadReg(part_ra, part_size, part_vi.signedness(isel), base_ra, opts.offset);
 9619                return true;
 9620            }
 9621            var used = false;
 9622            while (part_it.next()) |part_vi| used |= try part_vi.load(isel, root_ty, base_ra, .{
 9623                .root_vi = root_vi,
 9624                .offset = opts.offset + part_vi.get(isel).offset_from_parent,
 9625                .@"volatile" = opts.@"volatile",
 9626                .split = opts.split,
 9627                .wrap = switch (part_it.remaining) {
 9628                    else => null,
 9629                    0 => if (opts.wrap) |wrap| .{
 9630                        .signedness = wrap.signedness,
 9631                        .bits = @intCast(wrap.bits - 8 * part_vi.position(isel)[0]),
 9632                    } else null,
 9633                },
 9634                .expected_live_registers = opts.expected_live_registers,
 9635            });
 9636            return used;
 9637        }
 9638
 9639        fn store(
 9640            vi: Value.Index,
 9641            isel: *Select,
 9642            root_ty: ZigType,
 9643            base_ra: Register.Alias,
 9644            opts: MemoryAccessOptions,
 9645        ) !void {
 9646            const root_vi = switch (opts.root_vi) {
 9647                _ => |root_vi| root_vi,
 9648                .allocating => unreachable,
 9649                .free => vi,
 9650            };
 9651            var part_it = vi.parts(isel);
 9652            if (part_it.only()) |part_vi| only: {
 9653                const part_size = part_vi.size(isel);
 9654                const part_is_vector = part_vi.isVector(isel);
 9655                if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
 9656                    if (!opts.split) return;
 9657                    var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
 9658                    _ = try subpart_it.next(isel);
 9659                    part_it = vi.parts(isel);
 9660                    assert(part_it.only() == null);
 9661                    break :only;
 9662                }
 9663                const part_mat = try part_vi.matReg(isel);
 9664                try isel.storeReg(part_mat.ra, part_size, base_ra, opts.offset);
 9665                return part_mat.finish(isel);
 9666            }
 9667            while (part_it.next()) |part_vi| try part_vi.store(isel, root_ty, base_ra, .{
 9668                .root_vi = root_vi,
 9669                .offset = opts.offset + part_vi.get(isel).offset_from_parent,
 9670                .@"volatile" = opts.@"volatile",
 9671                .split = opts.split,
 9672                .wrap = switch (part_it.remaining) {
 9673                    else => null,
 9674                    0 => if (opts.wrap) |wrap| .{
 9675                        .signedness = wrap.signedness,
 9676                        .bits = @intCast(wrap.bits - 8 * part_vi.position(isel)[0]),
 9677                    } else null,
 9678                },
 9679                .expected_live_registers = opts.expected_live_registers,
 9680            });
 9681        }
 9682
 9683        fn mat(vi: Value.Index, isel: *Select) !void {
 9684            if (false) {
 9685                var part_it: Value.PartIterator = if (vi.size(isel) > 8) vi.parts(isel) else .initOne(vi);
 9686                if (part_it.only()) |part_vi| only: {
 9687                    const mat_ra = mat_ra: {
 9688                        if (part_vi.register(isel)) |mat_ra| {
 9689                            part_vi.get(isel).location_payload.small.register = .zr;
 9690                            const live_vi = isel.live_registers.getPtr(mat_ra);
 9691                            assert(live_vi.* == part_vi);
 9692                            live_vi.* = .allocating;
 9693                            break :mat_ra mat_ra;
 9694                        }
 9695                        if (part_vi.hint(isel)) |hint_ra| {
 9696                            const live_vi = isel.live_registers.getPtr(hint_ra);
 9697                            if (live_vi.* == .free) {
 9698                                live_vi.* = .allocating;
 9699                                isel.saved_registers.insert(hint_ra);
 9700                                break :mat_ra hint_ra;
 9701                            }
 9702                        }
 9703                        const part_size = part_vi.size(isel);
 9704                        const part_is_vector = part_vi.isVector(isel);
 9705                        if (part_size <= @as(@TypeOf(part_size), if (part_is_vector) 16 else 8))
 9706                            switch (if (part_is_vector) isel.tryAllocVecReg() else isel.tryAllocIntReg()) {
 9707                                .allocated => |ra| break :mat_ra ra,
 9708                                .fill_candidate, .out_of_registers => {},
 9709                            };
 9710                        _, const parent_vi = vi.valueParent(isel);
 9711                        switch (parent_vi.parent(isel)) {
 9712                            .unallocated => parent_vi.setParent(isel, .{ .stack_slot = parent_vi.allocStackSlot(isel) }),
 9713                            else => {},
 9714                        }
 9715                        break :only;
 9716                    };
 9717                    assert(isel.live_registers.get(mat_ra) == .allocating);
 9718                    try Value.Materialize.finish(.{ .vi = part_vi, .ra = mat_ra }, isel);
 9719                } else while (part_it.next()) |part_vi| try part_vi.mat(isel);
 9720            } else {
 9721                _, const parent_vi = vi.valueParent(isel);
 9722                switch (parent_vi.parent(isel)) {
 9723                    .unallocated => parent_vi.setParent(isel, .{ .stack_slot = parent_vi.allocStackSlot(isel) }),
 9724                    else => {},
 9725                }
 9726            }
 9727        }
 9728
 9729        fn matReg(vi: Value.Index, isel: *Select) !Value.Materialize {
 9730            const mat_ra = mat_ra: {
 9731                if (vi.register(isel)) |mat_ra| {
 9732                    vi.get(isel).location_payload.small.register = .zr;
 9733                    const live_vi = isel.live_registers.getPtr(mat_ra);
 9734                    assert(live_vi.* == vi);
 9735                    live_vi.* = .allocating;
 9736                    break :mat_ra mat_ra;
 9737                }
 9738                if (vi.hint(isel)) |hint_ra| {
 9739                    const live_vi = isel.live_registers.getPtr(hint_ra);
 9740                    if (live_vi.* == .free) {
 9741                        live_vi.* = .allocating;
 9742                        isel.saved_registers.insert(hint_ra);
 9743                        break :mat_ra hint_ra;
 9744                    }
 9745                }
 9746                break :mat_ra if (vi.isVector(isel)) try isel.allocVecReg() else try isel.allocIntReg();
 9747            };
 9748            assert(isel.live_registers.get(mat_ra) == .allocating);
 9749            return .{ .vi = vi, .ra = mat_ra };
 9750        }
 9751
 9752        fn defAddr(
 9753            def_vi: Value.Index,
 9754            isel: *Select,
 9755            root_ty: ZigType,
 9756            opts: struct {
 9757                root_vi: Value.Index = .free,
 9758                wrap: ?std.builtin.Type.Int = null,
 9759                expected_live_registers: *const LiveRegisters = &.initFill(.free),
 9760            },
 9761        ) !?void {
 9762            if (!def_vi.isUsed(isel)) return null;
 9763            const offset_from_parent: i65, const parent_vi = def_vi.valueParent(isel);
 9764            const stack_slot, const allocated = switch (parent_vi.parent(isel)) {
 9765                .unallocated => .{ parent_vi.allocStackSlot(isel), true },
 9766                .stack_slot => |stack_slot| .{ stack_slot, false },
 9767                else => unreachable,
 9768            };
 9769            _ = try def_vi.load(isel, root_ty, stack_slot.base, .{
 9770                .root_vi = opts.root_vi,
 9771                .offset = @intCast(stack_slot.offset + offset_from_parent),
 9772                .split = false,
 9773                .wrap = opts.wrap,
 9774                .expected_live_registers = opts.expected_live_registers,
 9775            });
 9776            if (allocated) parent_vi.setParent(isel, .{ .stack_slot = stack_slot });
 9777        }
 9778
 9779        fn defReg(def_vi: Value.Index, isel: *Select) !?Register.Alias {
 9780            var vi = def_vi;
 9781            var offset: i65 = 0;
 9782            var def_ra: ?Register.Alias = null;
 9783            while (true) {
 9784                if (vi.register(isel)) |ra| {
 9785                    vi.get(isel).location_payload.small.register = .zr;
 9786                    const live_vi = isel.live_registers.getPtr(ra);
 9787                    assert(live_vi.* == vi);
 9788                    if (def_ra == null and vi != def_vi) {
 9789                        var part_it = vi.parts(isel);
 9790                        assert(part_it.only() == null);
 9791
 9792                        const first_part_vi = part_it.next().?;
 9793                        const first_part_value = first_part_vi.get(isel);
 9794                        assert(first_part_value.offset_from_parent == 0);
 9795                        first_part_value.location_payload.small.register = ra;
 9796                        live_vi.* = first_part_vi;
 9797
 9798                        const vi_size = vi.size(isel);
 9799                        while (part_it.next()) |part_vi| {
 9800                            const part_offset, const part_size = part_vi.position(isel);
 9801                            const part_mat = try part_vi.matReg(isel);
 9802                            try isel.emit(if (part_vi.isVector(isel)) emit: {
 9803                                assert(part_offset == 0 and part_size == vi_size);
 9804                                break :emit switch (vi_size) {
 9805                                    else => unreachable,
 9806                                    2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
 9807                                        .fmov(ra.h(), .{ .register = part_mat.ra.h() })
 9808                                    else
 9809                                        .dup(ra.h(), part_mat.ra.@"h[]"(0)),
 9810                                    4 => .fmov(ra.s(), .{ .register = part_mat.ra.s() }),
 9811                                    8 => .fmov(ra.d(), .{ .register = part_mat.ra.d() }),
 9812                                    16 => .orr(ra.@"16b"(), part_mat.ra.@"16b"(), .{ .register = part_mat.ra.@"16b"() }),
 9813                                };
 9814                            } else switch (vi_size) {
 9815                                else => unreachable,
 9816                                1...4 => .bfm(ra.w(), part_mat.ra.w(), .{
 9817                                    .N = .word,
 9818                                    .immr = @as(u5, @truncate(32 - 8 * part_offset)),
 9819                                    .imms = @intCast(8 * part_size - 1),
 9820                                }),
 9821                                5...8 => .bfm(ra.x(), part_mat.ra.x(), .{
 9822                                    .N = .doubleword,
 9823                                    .immr = @as(u6, @truncate(64 - 8 * part_offset)),
 9824                                    .imms = @intCast(8 * part_size - 1),
 9825                                }),
 9826                            });
 9827                            try part_mat.finish(isel);
 9828                        }
 9829                        vi = def_vi;
 9830                        offset = 0;
 9831                        continue;
 9832                    }
 9833                    live_vi.* = .free;
 9834                    def_ra = ra;
 9835                }
 9836                offset += vi.get(isel).offset_from_parent;
 9837                switch (vi.parent(isel)) {
 9838                    else => unreachable,
 9839                    .unallocated => return def_ra,
 9840                    .stack_slot => |stack_slot| {
 9841                        offset += stack_slot.offset;
 9842                        const def_is_vector = def_vi.isVector(isel);
 9843                        const ra = def_ra orelse if (def_is_vector) try isel.allocVecReg() else try isel.allocIntReg();
 9844                        defer if (def_ra == null) isel.freeReg(ra);
 9845                        try isel.storeReg(ra, def_vi.size(isel), stack_slot.base, offset);
 9846                        return ra;
 9847                    },
 9848                    .value => |parent_vi| vi = parent_vi,
 9849                }
 9850            }
 9851        }
 9852
 9853        pub fn defUndef(def_vi: Value.Index, isel: *Select, root_ty: ZigType, opts: struct {
 9854            root_vi: Value.Index = .free,
 9855            offset: u64 = 0,
 9856            split: bool = true,
 9857        }) !void {
 9858            const root_vi = switch (opts.root_vi) {
 9859                _ => |root_vi| root_vi,
 9860                .allocating => unreachable,
 9861                .free => def_vi,
 9862            };
 9863            var part_it = def_vi.parts(isel);
 9864            if (part_it.only()) |part_vi| only: {
 9865                const part_size = part_vi.size(isel);
 9866                const part_is_vector = part_vi.isVector(isel);
 9867                if (part_size > @as(@TypeOf(part_size), if (part_is_vector) 16 else 8)) {
 9868                    if (!opts.split) return;
 9869                    var subpart_it = root_vi.field(root_ty, opts.offset, part_size - 1);
 9870                    _ = try subpart_it.next(isel);
 9871                    part_it = def_vi.parts(isel);
 9872                    assert(part_it.only() == null);
 9873                    break :only;
 9874                }
 9875                return if (try part_vi.defReg(isel)) |part_ra| try isel.emit(if (part_is_vector)
 9876                    .movi(switch (part_size) {
 9877                        else => unreachable,
 9878                        1...8 => part_ra.@"8b"(),
 9879                        9...16 => part_ra.@"16b"(),
 9880                    }, 0xaa, .{ .lsl = 0 })
 9881                else switch (part_size) {
 9882                    else => unreachable,
 9883                    1...4 => .orr(part_ra.w(), .wzr, .{ .immediate = .{
 9884                        .N = .word,
 9885                        .immr = 0b000001,
 9886                        .imms = 0b111100,
 9887                    } }),
 9888                    5...8 => .orr(part_ra.x(), .xzr, .{ .immediate = .{
 9889                        .N = .word,
 9890                        .immr = 0b000001,
 9891                        .imms = 0b111100,
 9892                    } }),
 9893                });
 9894            }
 9895            while (part_it.next()) |part_vi| try part_vi.defUndef(isel, root_ty, .{
 9896                .root_vi = root_vi,
 9897            });
 9898        }
 9899
 9900        pub fn liveIn(
 9901            vi: Value.Index,
 9902            isel: *Select,
 9903            src_ra: Register.Alias,
 9904            expected_live_registers: *const LiveRegisters,
 9905        ) !void {
 9906            const src_live_vi = isel.live_registers.getPtr(src_ra);
 9907            if (vi.register(isel)) |dst_ra| {
 9908                const dst_live_vi = isel.live_registers.getPtr(dst_ra);
 9909                assert(dst_live_vi.* == vi);
 9910                if (dst_ra == src_ra) {
 9911                    src_live_vi.* = .allocating;
 9912                    return;
 9913                }
 9914                dst_live_vi.* = .allocating;
 9915                if (try isel.fill(src_ra)) {
 9916                    assert(src_live_vi.* == .free);
 9917                    src_live_vi.* = .allocating;
 9918                }
 9919                assert(src_live_vi.* == .allocating);
 9920                try isel.emit(switch (dst_ra.isVector()) {
 9921                    false => switch (src_ra.isVector()) {
 9922                        false => switch (vi.size(isel)) {
 9923                            else => unreachable,
 9924                            1...4 => .orr(dst_ra.w(), .wzr, .{ .register = src_ra.w() }),
 9925                            5...8 => .orr(dst_ra.x(), .xzr, .{ .register = src_ra.x() }),
 9926                        },
 9927                        true => switch (vi.size(isel)) {
 9928                            else => unreachable,
 9929                            2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
 9930                                .fmov(dst_ra.w(), .{ .register = src_ra.h() })
 9931                            else
 9932                                .umov(dst_ra.w(), src_ra.@"h[]"(0)),
 9933                            4 => .fmov(dst_ra.w(), .{ .register = src_ra.s() }),
 9934                            8 => .fmov(dst_ra.x(), .{ .register = src_ra.d() }),
 9935                        },
 9936                    },
 9937                    true => switch (src_ra.isVector()) {
 9938                        false => size: switch (vi.size(isel)) {
 9939                            else => unreachable,
 9940                            2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
 9941                                .fmov(dst_ra.h(), .{ .register = src_ra.w() })
 9942                            else
 9943                                continue :size 4,
 9944                            4 => .fmov(dst_ra.s(), .{ .register = src_ra.w() }),
 9945                            8 => .fmov(dst_ra.d(), .{ .register = src_ra.x() }),
 9946                        },
 9947                        true => switch (vi.size(isel)) {
 9948                            else => unreachable,
 9949                            2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
 9950                                .fmov(dst_ra.h(), .{ .register = src_ra.h() })
 9951                            else
 9952                                .dup(dst_ra.h(), src_ra.@"h[]"(0)),
 9953                            4 => .fmov(dst_ra.s(), .{ .register = src_ra.s() }),
 9954                            8 => .fmov(dst_ra.d(), .{ .register = src_ra.d() }),
 9955                            16 => .orr(dst_ra.@"16b"(), src_ra.@"16b"(), .{ .register = src_ra.@"16b"() }),
 9956                        },
 9957                    },
 9958                });
 9959                assert(dst_live_vi.* == .allocating);
 9960                dst_live_vi.* = switch (expected_live_registers.get(dst_ra)) {
 9961                    _ => .allocating,
 9962                    .allocating => .allocating,
 9963                    .free => .free,
 9964                };
 9965            } else if (try isel.fill(src_ra)) {
 9966                assert(src_live_vi.* == .free);
 9967                src_live_vi.* = .allocating;
 9968            }
 9969            assert(src_live_vi.* == .allocating);
 9970            vi.get(isel).location_payload.small.register = src_ra;
 9971        }
 9972
 9973        pub fn defLiveIn(
 9974            vi: Value.Index,
 9975            isel: *Select,
 9976            src_ra: Register.Alias,
 9977            expected_live_registers: *const LiveRegisters,
 9978        ) !void {
 9979            try vi.liveIn(isel, src_ra, expected_live_registers);
 9980            const offset_from_parent, const parent_vi = vi.valueParent(isel);
 9981            switch (parent_vi.parent(isel)) {
 9982                .unallocated => {},
 9983                .stack_slot => |stack_slot| if (stack_slot.base != Register.Alias.fp) try isel.storeReg(
 9984                    src_ra,
 9985                    vi.size(isel),
 9986                    stack_slot.base,
 9987                    @as(i65, stack_slot.offset) + offset_from_parent,
 9988                ),
 9989                else => unreachable,
 9990            }
 9991            try vi.spillReg(isel, src_ra, 0, expected_live_registers);
 9992        }
 9993
 9994        fn spillReg(
 9995            vi: Value.Index,
 9996            isel: *Select,
 9997            src_ra: Register.Alias,
 9998            start_offset: u64,
 9999            expected_live_registers: *const LiveRegisters,
10000        ) !void {
10001            assert(isel.live_registers.get(src_ra) == .allocating);
10002            var part_it = vi.parts(isel);
10003            if (part_it.only()) |part_vi| {
10004                const dst_ra = part_vi.register(isel) orelse return;
10005                if (dst_ra == src_ra) return;
10006                const part_size = part_vi.size(isel);
10007                const part_ra = if (part_vi.isVector(isel)) try isel.allocIntReg() else dst_ra;
10008                defer if (part_ra != dst_ra) isel.freeReg(part_ra);
10009                if (part_ra != dst_ra) try isel.emit(part_size: switch (part_size) {
10010                    else => unreachable,
10011                    2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
10012                        .fmov(dst_ra.h(), .{ .register = part_ra.w() })
10013                    else
10014                        continue :part_size 4,
10015                    4 => .fmov(dst_ra.s(), .{ .register = part_ra.w() }),
10016                    8 => .fmov(dst_ra.d(), .{ .register = part_ra.x() }),
10017                });
10018                try isel.emit(switch (start_offset + part_size) {
10019                    else => unreachable,
10020                    1...4 => |end_offset| switch (part_vi.signedness(isel)) {
10021                        .signed => .sbfm(part_ra.w(), src_ra.w(), .{
10022                            .N = .word,
10023                            .immr = @intCast(8 * start_offset),
10024                            .imms = @intCast(8 * end_offset - 1),
10025                        }),
10026                        .unsigned => .ubfm(part_ra.w(), src_ra.w(), .{
10027                            .N = .word,
10028                            .immr = @intCast(8 * start_offset),
10029                            .imms = @intCast(8 * end_offset - 1),
10030                        }),
10031                    },
10032                    5...8 => |end_offset| switch (part_vi.signedness(isel)) {
10033                        .signed => .sbfm(part_ra.x(), src_ra.x(), .{
10034                            .N = .doubleword,
10035                            .immr = @intCast(8 * start_offset),
10036                            .imms = @intCast(8 * end_offset - 1),
10037                        }),
10038                        .unsigned => .ubfm(part_ra.x(), src_ra.x(), .{
10039                            .N = .doubleword,
10040                            .immr = @intCast(8 * start_offset),
10041                            .imms = @intCast(8 * end_offset - 1),
10042                        }),
10043                    },
10044                });
10045                const value_ra = &part_vi.get(isel).location_payload.small.register;
10046                assert(value_ra.* == dst_ra);
10047                value_ra.* = .zr;
10048                const dst_live_vi = isel.live_registers.getPtr(dst_ra);
10049                assert(dst_live_vi.* == part_vi);
10050                dst_live_vi.* = switch (expected_live_registers.get(dst_ra)) {
10051                    _ => .allocating,
10052                    .allocating => unreachable,
10053                    .free => .free,
10054                };
10055            } else while (part_it.next()) |part_vi| try part_vi.spillReg(
10056                isel,
10057                src_ra,
10058                start_offset + part_vi.get(isel).offset_from_parent,
10059                expected_live_registers,
10060            );
10061        }
10062
10063        fn liveOut(vi: Value.Index, isel: *Select, ra: Register.Alias) !void {
10064            assert(try isel.fill(ra));
10065            const live_vi = isel.live_registers.getPtr(ra);
10066            assert(live_vi.* == .free);
10067            live_vi.* = .allocating;
10068            try Value.Materialize.finish(.{ .vi = vi, .ra = ra }, isel);
10069        }
10070
10071        fn allocStackSlot(vi: Value.Index, isel: *Select) Value.Indirect {
10072            const offset = vi.alignment(isel).forward(isel.stack_size);
10073            isel.stack_size = @intCast(offset + vi.size(isel));
10074            tracking_log.debug("${d} -> [sp, #0x{x}]", .{ @intFromEnum(vi), @abs(offset) });
10075            return .{
10076                .base = .sp,
10077                .offset = @intCast(offset),
10078            };
10079        }
10080
10081        fn address(initial_vi: Value.Index, isel: *Select, initial_offset: u64, ptr_ra: Register.Alias) !void {
10082            var vi = initial_vi;
10083            var offset: i65 = vi.get(isel).offset_from_parent + initial_offset;
10084            parent: switch (vi.parent(isel)) {
10085                .unallocated => {
10086                    const stack_slot = vi.allocStackSlot(isel);
10087                    vi.setParent(isel, .{ .stack_slot = stack_slot });
10088                    continue :parent .{ .stack_slot = stack_slot };
10089                },
10090                .stack_slot => |stack_slot| {
10091                    offset += stack_slot.offset;
10092                    const lo12: u12 = @truncate(@abs(offset) >> 0);
10093                    const hi12: u12 = @intCast(@abs(offset) >> 12);
10094                    if (hi12 > 0) try isel.emit(if (offset >= 0) .add(
10095                        ptr_ra.x(),
10096                        if (lo12 > 0) ptr_ra.x() else stack_slot.base.x(),
10097                        .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
10098                    ) else .sub(
10099                        ptr_ra.x(),
10100                        if (lo12 > 0) ptr_ra.x() else stack_slot.base.x(),
10101                        .{ .shifted_immediate = .{ .immediate = hi12, .lsl = .@"12" } },
10102                    ));
10103                    if (lo12 > 0 or hi12 == 0) try isel.emit(if (offset >= 0) .add(
10104                        ptr_ra.x(),
10105                        stack_slot.base.x(),
10106                        .{ .immediate = lo12 },
10107                    ) else .sub(
10108                        ptr_ra.x(),
10109                        stack_slot.base.x(),
10110                        .{ .immediate = lo12 },
10111                    ));
10112                },
10113                .address => |address_vi| try address_vi.liveOut(isel, ptr_ra),
10114                .value => |parent_vi| {
10115                    vi = parent_vi;
10116                    offset += vi.get(isel).offset_from_parent;
10117                    continue :parent vi.parent(isel);
10118                },
10119                .constant => |constant| {
10120                    const pt = isel.pt;
10121                    const zcu = pt.zcu;
10122                    switch (true) {
10123                        false => {
10124                            try isel.uav_relocs.append(zcu.gpa, .{
10125                                .uav = .{
10126                                    .val = constant.toIntern(),
10127                                    .orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10128                                },
10129                                .reloc = .{
10130                                    .label = @intCast(isel.instructions.items.len),
10131                                    .addend = @intCast(offset),
10132                                },
10133                            });
10134                            try isel.emit(.adr(ptr_ra.x(), 0));
10135                        },
10136                        true => {
10137                            try isel.uav_relocs.append(zcu.gpa, .{
10138                                .uav = .{
10139                                    .val = constant.toIntern(),
10140                                    .orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10141                                },
10142                                .reloc = .{
10143                                    .label = @intCast(isel.instructions.items.len),
10144                                    .addend = @intCast(offset),
10145                                },
10146                            });
10147                            try isel.emit(.add(ptr_ra.x(), ptr_ra.x(), .{ .immediate = 0 }));
10148                            try isel.uav_relocs.append(zcu.gpa, .{
10149                                .uav = .{
10150                                    .val = constant.toIntern(),
10151                                    .orig_ty = (try pt.singleConstPtrType(constant.typeOf(zcu))).toIntern(),
10152                                },
10153                                .reloc = .{
10154                                    .label = @intCast(isel.instructions.items.len),
10155                                    .addend = @intCast(offset),
10156                                },
10157                            });
10158                            try isel.emit(.adrp(ptr_ra.x(), 0));
10159                        },
10160                    }
10161                },
10162            }
10163        }
10164    };
10165
10166    pub const PartIterator = struct {
10167        vi: Value.Index,
10168        remaining: Value.PartsLen,
10169
10170        fn initOne(vi: Value.Index) PartIterator {
10171            return .{ .vi = vi, .remaining = 1 };
10172        }
10173
10174        pub fn next(it: *PartIterator) ?Value.Index {
10175            if (it.remaining == 0) return null;
10176            it.remaining -= 1;
10177            defer it.vi = @enumFromInt(@intFromEnum(it.vi) + 1);
10178            return it.vi;
10179        }
10180
10181        pub fn peek(it: PartIterator) ?Value.Index {
10182            var it_mut = it;
10183            return it_mut.next();
10184        }
10185
10186        pub fn only(it: PartIterator) ?Value.Index {
10187            return if (it.remaining == 1) it.vi else null;
10188        }
10189    };
10190
10191    const FieldPartIterator = struct {
10192        vi: Value.Index,
10193        ty: ZigType,
10194        field_offset: u64,
10195        field_size: u64,
10196        next_offset: u64,
10197
10198        fn next(it: *FieldPartIterator, isel: *Select) !?struct { offset: u64, vi: Value.Index } {
10199            const next_offset = it.next_offset;
10200            const next_part_size = it.field_size - next_offset;
10201            if (next_part_size == 0) return null;
10202            var next_part_offset = it.field_offset + next_offset;
10203
10204            const zcu = isel.pt.zcu;
10205            const ip = &zcu.intern_pool;
10206            var vi = it.vi;
10207            var ty = it.ty;
10208            var ty_size = vi.size(isel);
10209            assert(ty_size == ty.abiSize(zcu));
10210            var offset: u64 = 0;
10211            var size = ty_size;
10212            assert(next_part_offset + next_part_size <= size);
10213            while (next_part_offset > 0 or next_part_size < size) {
10214                const part_vi = vi.partAtOffset(isel, next_part_offset);
10215                if (part_vi != vi) {
10216                    vi = part_vi;
10217                    const part_offset, size = part_vi.position(isel);
10218                    assert(part_offset <= next_part_offset and part_offset + size > next_part_offset);
10219                    offset += part_offset;
10220                    next_part_offset -= part_offset;
10221                    continue;
10222                }
10223                try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
10224                type_key: switch (ip.indexToKey(ty.toIntern())) {
10225                    else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10226                    .int_type => |int_type| switch (int_type.bits) {
10227                        0 => unreachable,
10228                        1...64 => unreachable,
10229                        65...256 => |bits| if (offset == 0 and size == ty_size) {
10230                            const parts_len = std.math.divCeil(u16, bits, 64) catch unreachable;
10231                            vi.setParts(isel, @intCast(parts_len));
10232                            for (0..parts_len) |part_index| _ = vi.addPart(isel, 8 * part_index, 8);
10233                        },
10234                        else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10235                    },
10236                    .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
10237                        .one, .many, .c => unreachable,
10238                        .slice => if (offset == 0 and size == ty_size) {
10239                            vi.setParts(isel, 2);
10240                            _ = vi.addPart(isel, 0, 8);
10241                            _ = vi.addPart(isel, 8, 8);
10242                        } else unreachable,
10243                    },
10244                    .opt_type => |child_type| if (ty.optionalReprIsPayload(zcu)) continue :type_key ip.indexToKey(child_type) else {
10245                        const child_ty: ZigType = .fromInterned(child_type);
10246                        const child_size = child_ty.abiSize(zcu);
10247                        if (offset == 0 and size == child_size) {
10248                            ty = child_ty;
10249                            ty_size = child_size;
10250                            continue :type_key ip.indexToKey(child_type);
10251                        }
10252                        switch (child_size) {
10253                            0...8, 16 => if (offset == 0 and size == ty_size) {
10254                                vi.setParts(isel, 2);
10255                                _ = vi.addPart(isel, 0, child_size);
10256                                _ = vi.addPart(isel, child_size, 1);
10257                            } else unreachable,
10258                            9...15 => if (offset == 0 and size == ty_size) {
10259                                vi.setParts(isel, 2);
10260                                _ = vi.addPart(isel, 0, 8);
10261                                _ = vi.addPart(isel, 8, ty_size - 8);
10262                            } else if (offset == 8 and size == ty_size - 8) {
10263                                vi.setParts(isel, 2);
10264                                _ = vi.addPart(isel, 0, child_size - 8);
10265                                _ = vi.addPart(isel, child_size - 8, 1);
10266                            } else unreachable,
10267                            else => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10268                        }
10269                    },
10270                    .array_type => |array_type| {
10271                        const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10272                        const array_len = array_type.lenIncludingSentinel();
10273                        if (array_len > Value.max_parts and
10274                            (std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
10275                            return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10276                        const alignment = vi.alignment(isel);
10277                        const Part = struct { offset: u64, size: u64 };
10278                        var parts: [Value.max_parts]Part = undefined;
10279                        var parts_len: Value.PartsLen = 0;
10280                        const elem_ty: ZigType = .fromInterned(array_type.child);
10281                        const elem_size = elem_ty.abiSize(zcu);
10282                        const elem_signedness = if (ty.isAbiInt(zcu)) elem_signedness: {
10283                            const elem_int_info = elem_ty.intInfo(zcu);
10284                            break :elem_signedness if (elem_int_info.bits <= 16) elem_int_info.signedness else null;
10285                        } else null;
10286                        const elem_is_vector = elem_size <= 16 and
10287                            CallAbiIterator.homogeneousAggregateBaseType(zcu, elem_ty.toIntern()) != null;
10288                        var elem_end: u64 = 0;
10289                        for (0..@intCast(array_len)) |_| {
10290                            const elem_begin = elem_end;
10291                            if (elem_begin >= offset + size) break;
10292                            elem_end = elem_begin + elem_size;
10293                            if (elem_end <= offset) continue;
10294                            if (offset >= elem_begin and offset + size <= elem_begin + elem_size) {
10295                                ty = elem_ty;
10296                                ty_size = elem_size;
10297                                offset -= elem_begin;
10298                                continue :type_key ip.indexToKey(elem_ty.toIntern());
10299                            }
10300                            if (parts_len > 0) combine: {
10301                                const prev_part = &parts[parts_len - 1];
10302                                const combined_size = elem_end - prev_part.offset;
10303                                if (combined_size > @as(u64, 1) << @min(
10304                                    min_part_log2_stride,
10305                                    alignment.toLog2Units(),
10306                                    @ctz(prev_part.offset),
10307                                )) break :combine;
10308                                prev_part.size = combined_size;
10309                                continue;
10310                            }
10311                            parts[parts_len] = .{ .offset = elem_begin, .size = elem_size };
10312                            parts_len += 1;
10313                        }
10314                        vi.setParts(isel, parts_len);
10315                        for (parts[0..parts_len]) |part| {
10316                            const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10317                            if (elem_signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10318                            if (elem_is_vector) subpart_vi.setIsVector(isel);
10319                        }
10320                    },
10321                    .anyframe_type => unreachable,
10322                    .error_union_type => |error_union_type| {
10323                        const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10324                        if ((std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
10325                            return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10326                        const alignment = vi.alignment(isel);
10327                        const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
10328                        const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
10329                        const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
10330                        const Part = struct { offset: u64, size: u64, signedness: ?std.builtin.Signedness, is_vector: bool };
10331                        var parts: [2]Part = undefined;
10332                        var parts_len: Value.PartsLen = 0;
10333                        var field_end: u64 = 0;
10334                        for (0..2) |field_index| {
10335                            const field_ty: ZigType, const field_begin = switch (@as(enum { error_set, payload }, switch (field_index) {
10336                                0 => if (error_set_offset < payload_offset) .error_set else .payload,
10337                                1 => if (error_set_offset < payload_offset) .payload else .error_set,
10338                                else => unreachable,
10339                            })) {
10340                                .error_set => .{ .fromInterned(error_union_type.error_set_type), error_set_offset },
10341                                .payload => .{ payload_ty, payload_offset },
10342                            };
10343                            if (field_begin >= offset + size) break;
10344                            const field_size = field_ty.abiSize(zcu);
10345                            if (field_size == 0) continue;
10346                            field_end = field_begin + field_size;
10347                            if (field_end <= offset) continue;
10348                            if (offset >= field_begin and offset + size <= field_begin + field_size) {
10349                                ty = field_ty;
10350                                ty_size = field_size;
10351                                offset -= field_begin;
10352                                continue :type_key ip.indexToKey(field_ty.toIntern());
10353                            }
10354                            const field_signedness = if (field_ty.isAbiInt(zcu)) field_signedness: {
10355                                const field_int_info = field_ty.intInfo(zcu);
10356                                break :field_signedness if (field_int_info.bits <= 16) field_int_info.signedness else null;
10357                            } else null;
10358                            const field_is_vector = field_size <= 16 and
10359                                CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10360                            if (parts_len > 0) combine: {
10361                                const prev_part = &parts[parts_len - 1];
10362                                const combined_size = field_end - prev_part.offset;
10363                                if (combined_size > @as(u64, 1) << @min(
10364                                    min_part_log2_stride,
10365                                    alignment.toLog2Units(),
10366                                    @ctz(prev_part.offset),
10367                                )) break :combine;
10368                                prev_part.size = combined_size;
10369                                prev_part.signedness = null;
10370                                prev_part.is_vector &= field_is_vector;
10371                                continue;
10372                            }
10373                            parts[parts_len] = .{
10374                                .offset = field_begin,
10375                                .size = field_size,
10376                                .signedness = field_signedness,
10377                                .is_vector = field_is_vector,
10378                            };
10379                            parts_len += 1;
10380                        }
10381                        vi.setParts(isel, parts_len);
10382                        for (parts[0..parts_len]) |part| {
10383                            const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10384                            if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10385                            if (part.is_vector) subpart_vi.setIsVector(isel);
10386                        }
10387                    },
10388                    .simple_type => |simple_type| switch (simple_type) {
10389                        .f16, .f32, .f64, .f128, .c_longdouble => return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)}),
10390                        .f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
10391                        .usize,
10392                        .isize,
10393                        .c_char,
10394                        .c_short,
10395                        .c_ushort,
10396                        .c_int,
10397                        .c_uint,
10398                        .c_long,
10399                        .c_ulong,
10400                        .c_longlong,
10401                        .c_ulonglong,
10402                        => continue :type_key .{ .int_type = ty.intInfo(zcu) },
10403                        .anyopaque,
10404                        .void,
10405                        .type,
10406                        .comptime_int,
10407                        .comptime_float,
10408                        .noreturn,
10409                        .null,
10410                        .undefined,
10411                        .enum_literal,
10412                        .adhoc_inferred_error_set,
10413                        .generic_poison,
10414                        => unreachable,
10415                        .bool => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 1 } },
10416                        .anyerror => continue :type_key .{ .int_type = .{
10417                            .signedness = .unsigned,
10418                            .bits = zcu.errorSetBits(),
10419                        } },
10420                    },
10421                    .struct_type => {
10422                        const loaded_struct = ip.loadStructType(ty.toIntern());
10423                        switch (loaded_struct.layout) {
10424                            .auto, .@"extern" => {},
10425                            .@"packed" => continue :type_key .{
10426                                .int_type = ip.indexToKey(loaded_struct.backingIntTypeUnordered(ip)).int_type,
10427                            },
10428                        }
10429                        const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10430                        if (loaded_struct.field_types.len > Value.max_parts and
10431                            (std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
10432                            return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10433                        const alignment = vi.alignment(isel);
10434                        const Part = struct { offset: u64, size: u64, signedness: ?std.builtin.Signedness, is_vector: bool };
10435                        var parts: [Value.max_parts]Part = undefined;
10436                        var parts_len: Value.PartsLen = 0;
10437                        var field_end: u64 = 0;
10438                        var field_it = loaded_struct.iterateRuntimeOrder(ip);
10439                        while (field_it.next()) |field_index| {
10440                            const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
10441                            const field_begin = switch (loaded_struct.fieldAlign(ip, field_index)) {
10442                                .none => field_ty.abiAlignment(zcu),
10443                                else => |field_align| field_align,
10444                            }.forward(field_end);
10445                            if (field_begin >= offset + size) break;
10446                            const field_size = field_ty.abiSize(zcu);
10447                            field_end = field_begin + field_size;
10448                            if (field_end <= offset) continue;
10449                            if (offset >= field_begin and offset + size <= field_begin + field_size) {
10450                                ty = field_ty;
10451                                ty_size = field_size;
10452                                offset -= field_begin;
10453                                continue :type_key ip.indexToKey(field_ty.toIntern());
10454                            }
10455                            const field_signedness = if (field_ty.isAbiInt(zcu)) field_signedness: {
10456                                const field_int_info = field_ty.intInfo(zcu);
10457                                break :field_signedness if (field_int_info.bits <= 16) field_int_info.signedness else null;
10458                            } else null;
10459                            const field_is_vector = field_size <= 16 and
10460                                CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10461                            if (parts_len > 0) combine: {
10462                                const prev_part = &parts[parts_len - 1];
10463                                const combined_size = field_end - prev_part.offset;
10464                                if (combined_size > @as(u64, 1) << @min(
10465                                    min_part_log2_stride,
10466                                    alignment.toLog2Units(),
10467                                    @ctz(prev_part.offset),
10468                                )) break :combine;
10469                                prev_part.size = combined_size;
10470                                prev_part.signedness = null;
10471                                prev_part.is_vector &= field_is_vector;
10472                                continue;
10473                            }
10474                            parts[parts_len] = .{
10475                                .offset = field_begin,
10476                                .size = field_size,
10477                                .signedness = field_signedness,
10478                                .is_vector = field_is_vector,
10479                            };
10480                            parts_len += 1;
10481                        }
10482                        vi.setParts(isel, parts_len);
10483                        for (parts[0..parts_len]) |part| {
10484                            const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10485                            if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10486                            if (part.is_vector) subpart_vi.setIsVector(isel);
10487                        }
10488                    },
10489                    .tuple_type => |tuple_type| {
10490                        const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10491                        if (tuple_type.types.len > Value.max_parts and
10492                            (std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
10493                            return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10494                        const alignment = vi.alignment(isel);
10495                        const Part = struct { offset: u64, size: u64, is_vector: bool };
10496                        var parts: [Value.max_parts]Part = undefined;
10497                        var parts_len: Value.PartsLen = 0;
10498                        var field_end: u64 = 0;
10499                        for (tuple_type.types.get(ip), tuple_type.values.get(ip)) |field_type, field_value| {
10500                            if (field_value != .none) continue;
10501                            const field_ty: ZigType = .fromInterned(field_type);
10502                            const field_begin = field_ty.abiAlignment(zcu).forward(field_end);
10503                            if (field_begin >= offset + size) break;
10504                            const field_size = field_ty.abiSize(zcu);
10505                            if (field_size == 0) continue;
10506                            field_end = field_begin + field_size;
10507                            if (field_end <= offset) continue;
10508                            if (offset >= field_begin and offset + size <= field_begin + field_size) {
10509                                ty = field_ty;
10510                                ty_size = field_size;
10511                                offset -= field_begin;
10512                                continue :type_key ip.indexToKey(field_ty.toIntern());
10513                            }
10514                            const field_is_vector = field_size <= 16 and
10515                                CallAbiIterator.homogeneousAggregateBaseType(zcu, field_ty.toIntern()) != null;
10516                            if (parts_len > 0) combine: {
10517                                const prev_part = &parts[parts_len - 1];
10518                                const combined_size = field_end - prev_part.offset;
10519                                if (combined_size > @as(u64, 1) << @min(
10520                                    min_part_log2_stride,
10521                                    alignment.toLog2Units(),
10522                                    @ctz(prev_part.offset),
10523                                )) break :combine;
10524                                prev_part.size = combined_size;
10525                                prev_part.is_vector &= field_is_vector;
10526                                continue;
10527                            }
10528                            parts[parts_len] = .{ .offset = field_begin, .size = field_size, .is_vector = field_is_vector };
10529                            parts_len += 1;
10530                        }
10531                        vi.setParts(isel, parts_len);
10532                        for (parts[0..parts_len]) |part| {
10533                            const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10534                            if (part.is_vector) subpart_vi.setIsVector(isel);
10535                        }
10536                    },
10537                    .union_type => {
10538                        const loaded_union = ip.loadUnionType(ty.toIntern());
10539                        switch (loaded_union.flagsUnordered(ip).layout) {
10540                            .auto, .@"extern" => {},
10541                            .@"packed" => continue :type_key .{ .int_type = .{
10542                                .signedness = .unsigned,
10543                                .bits = @intCast(ty.bitSize(zcu)),
10544                            } },
10545                        }
10546                        const min_part_log2_stride: u5 = if (size > 16) 4 else if (size > 8) 3 else 0;
10547                        if ((std.math.divCeil(u64, size, @as(u64, 1) << min_part_log2_stride) catch unreachable) > Value.max_parts)
10548                            return isel.fail("Value.FieldPartIterator.next({f})", .{isel.fmtType(ty)});
10549                        const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
10550                        const alignment = vi.alignment(isel);
10551                        const tag_offset = union_layout.tagOffset();
10552                        const payload_offset = union_layout.payloadOffset();
10553                        const Part = struct { offset: u64, size: u64, signedness: ?std.builtin.Signedness };
10554                        var parts: [2]Part = undefined;
10555                        var parts_len: Value.PartsLen = 0;
10556                        var field_end: u64 = 0;
10557                        for (0..2) |field_index| {
10558                            const field: enum { tag, payload } = switch (field_index) {
10559                                0 => if (tag_offset < payload_offset) .tag else .payload,
10560                                1 => if (tag_offset < payload_offset) .payload else .tag,
10561                                else => unreachable,
10562                            };
10563                            const field_size, const field_begin = switch (field) {
10564                                .tag => .{ union_layout.tag_size, tag_offset },
10565                                .payload => .{ union_layout.payload_size, payload_offset },
10566                            };
10567                            if (field_begin >= offset + size) break;
10568                            if (field_size == 0) continue;
10569                            field_end = field_begin + field_size;
10570                            if (field_end <= offset) continue;
10571                            const field_signedness = field_signedness: switch (field) {
10572                                .tag => {
10573                                    if (offset >= field_begin and offset + size <= field_begin + field_size) {
10574                                        ty = .fromInterned(loaded_union.enum_tag_ty);
10575                                        ty_size = field_size;
10576                                        offset -= field_begin;
10577                                        continue :type_key ip.indexToKey(loaded_union.enum_tag_ty);
10578                                    }
10579                                    break :field_signedness ip.indexToKey(loaded_union.loadTagType(ip).tag_ty).int_type.signedness;
10580                                },
10581                                .payload => null,
10582                            };
10583                            if (parts_len > 0) combine: {
10584                                const prev_part = &parts[parts_len - 1];
10585                                const combined_size = field_end - prev_part.offset;
10586                                if (combined_size > @as(u64, 1) << @min(
10587                                    min_part_log2_stride,
10588                                    alignment.toLog2Units(),
10589                                    @ctz(prev_part.offset),
10590                                )) break :combine;
10591                                prev_part.size = combined_size;
10592                                prev_part.signedness = null;
10593                                continue;
10594                            }
10595                            parts[parts_len] = .{
10596                                .offset = field_begin,
10597                                .size = field_size,
10598                                .signedness = field_signedness,
10599                            };
10600                            parts_len += 1;
10601                        }
10602                        vi.setParts(isel, parts_len);
10603                        for (parts[0..parts_len]) |part| {
10604                            const subpart_vi = vi.addPart(isel, part.offset - offset, part.size);
10605                            if (part.signedness) |signedness| subpart_vi.setSignedness(isel, signedness);
10606                        }
10607                    },
10608                    .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
10609                    .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty),
10610                    .error_set_type,
10611                    .inferred_error_set_type,
10612                    => continue :type_key .{ .simple_type = .anyerror },
10613                    .undef,
10614                    .simple_value,
10615                    .variable,
10616                    .@"extern",
10617                    .func,
10618                    .int,
10619                    .err,
10620                    .error_union,
10621                    .enum_literal,
10622                    .enum_tag,
10623                    .empty_enum_value,
10624                    .float,
10625                    .ptr,
10626                    .slice,
10627                    .opt,
10628                    .aggregate,
10629                    .un,
10630                    .memoized_call,
10631                    => unreachable, // values, not types
10632                }
10633            }
10634            it.next_offset = next_offset + size;
10635            return .{ .offset = next_part_offset - next_offset, .vi = vi };
10636        }
10637
10638        fn only(it: *FieldPartIterator, isel: *Select) !?Value.Index {
10639            const part = try it.next(isel);
10640            assert(part.?.offset == 0);
10641            return if (try it.next(isel)) |_| null else part.?.vi;
10642        }
10643    };
10644
10645    const Materialize = struct {
10646        vi: Value.Index,
10647        ra: Register.Alias,
10648
10649        fn finish(mat: Value.Materialize, isel: *Select) error{ OutOfMemory, CodegenFail }!void {
10650            const live_vi = isel.live_registers.getPtr(mat.ra);
10651            assert(live_vi.* == .allocating);
10652            var vi = mat.vi;
10653            var offset: u64 = 0;
10654            const size = mat.vi.size(isel);
10655            free: while (true) {
10656                if (vi.register(isel)) |ra| {
10657                    if (ra != mat.ra) break :free try isel.emit(if (vi == mat.vi) if (mat.ra.isVector()) switch (size) {
10658                        else => unreachable,
10659                        2 => if (isel.target.cpu.has(.aarch64, .fullfp16))
10660                            .fmov(mat.ra.h(), .{ .register = ra.h() })
10661                        else
10662                            .dup(mat.ra.h(), ra.@"h[]"(0)),
10663                        4 => .fmov(mat.ra.s(), .{ .register = ra.s() }),
10664                        8 => .fmov(mat.ra.d(), .{ .register = ra.d() }),
10665                        16 => .orr(mat.ra.@"16b"(), ra.@"16b"(), .{ .register = ra.@"16b"() }),
10666                    } else switch (size) {
10667                        else => unreachable,
10668                        1...4 => .orr(mat.ra.w(), .wzr, .{ .register = ra.w() }),
10669                        5...8 => .orr(mat.ra.x(), .xzr, .{ .register = ra.x() }),
10670                    } else switch (offset + size) {
10671                        else => unreachable,
10672                        1...4 => |end_offset| switch (mat.vi.signedness(isel)) {
10673                            .signed => .sbfm(mat.ra.w(), ra.w(), .{
10674                                .N = .word,
10675                                .immr = @intCast(8 * offset),
10676                                .imms = @intCast(8 * end_offset - 1),
10677                            }),
10678                            .unsigned => .ubfm(mat.ra.w(), ra.w(), .{
10679                                .N = .word,
10680                                .immr = @intCast(8 * offset),
10681                                .imms = @intCast(8 * end_offset - 1),
10682                            }),
10683                        },
10684                        5...8 => |end_offset| switch (mat.vi.signedness(isel)) {
10685                            .signed => .sbfm(mat.ra.x(), ra.x(), .{
10686                                .N = .doubleword,
10687                                .immr = @intCast(8 * offset),
10688                                .imms = @intCast(8 * end_offset - 1),
10689                            }),
10690                            .unsigned => .ubfm(mat.ra.x(), ra.x(), .{
10691                                .N = .doubleword,
10692                                .immr = @intCast(8 * offset),
10693                                .imms = @intCast(8 * end_offset - 1),
10694                            }),
10695                        },
10696                    });
10697                    mat.vi.get(isel).location_payload.small.register = mat.ra;
10698                    live_vi.* = mat.vi;
10699                    return;
10700                }
10701                offset += vi.get(isel).offset_from_parent;
10702                switch (vi.parent(isel)) {
10703                    .unallocated => {
10704                        mat.vi.get(isel).location_payload.small.register = mat.ra;
10705                        live_vi.* = mat.vi;
10706                        return;
10707                    },
10708                    .stack_slot => |stack_slot| break :free try isel.loadReg(
10709                        mat.ra,
10710                        size,
10711                        mat.vi.signedness(isel),
10712                        stack_slot.base,
10713                        @as(i65, stack_slot.offset) + offset,
10714                    ),
10715                    .address => |base_vi| {
10716                        const base_mat = try base_vi.matReg(isel);
10717                        try isel.loadReg(mat.ra, size, mat.vi.signedness(isel), base_mat.ra, offset);
10718                        break :free try base_mat.finish(isel);
10719                    },
10720                    .value => |parent_vi| vi = parent_vi,
10721                    .constant => |initial_constant| {
10722                        const zcu = isel.pt.zcu;
10723                        const ip = &zcu.intern_pool;
10724                        var constant = initial_constant.toIntern();
10725                        var constant_key = ip.indexToKey(constant);
10726                        while (true) {
10727                            constant_key: switch (constant_key) {
10728                                .int_type,
10729                                .ptr_type,
10730                                .array_type,
10731                                .vector_type,
10732                                .opt_type,
10733                                .anyframe_type,
10734                                .error_union_type,
10735                                .simple_type,
10736                                .struct_type,
10737                                .tuple_type,
10738                                .union_type,
10739                                .opaque_type,
10740                                .enum_type,
10741                                .func_type,
10742                                .error_set_type,
10743                                .inferred_error_set_type,
10744
10745                                .enum_literal,
10746                                .empty_enum_value,
10747                                .memoized_call,
10748                                => unreachable, // not a runtime value
10749                                .undef => break :free try isel.emit(if (mat.ra.isVector()) .movi(switch (size) {
10750                                    else => unreachable,
10751                                    1...8 => mat.ra.@"8b"(),
10752                                    9...16 => mat.ra.@"16b"(),
10753                                }, 0xaa, .{ .lsl = 0 }) else switch (size) {
10754                                    else => unreachable,
10755                                    1...4 => .orr(mat.ra.w(), .wzr, .{ .immediate = .{
10756                                        .N = .word,
10757                                        .immr = 0b000001,
10758                                        .imms = 0b111100,
10759                                    } }),
10760                                    5...8 => .orr(mat.ra.x(), .xzr, .{ .immediate = .{
10761                                        .N = .word,
10762                                        .immr = 0b000001,
10763                                        .imms = 0b111100,
10764                                    } }),
10765                                }),
10766                                .simple_value => |simple_value| switch (simple_value) {
10767                                    .undefined, .void, .null, .empty_tuple, .@"unreachable" => unreachable,
10768                                    .true => continue :constant_key .{ .int = .{
10769                                        .ty = .bool_type,
10770                                        .storage = .{ .u64 = 1 },
10771                                    } },
10772                                    .false => continue :constant_key .{ .int = .{
10773                                        .ty = .bool_type,
10774                                        .storage = .{ .u64 = 0 },
10775                                    } },
10776                                },
10777                                .int => |int| break :free storage: switch (int.storage) {
10778                                    .u64 => |imm| try isel.movImmediate(switch (size) {
10779                                        else => unreachable,
10780                                        1...4 => mat.ra.w(),
10781                                        5...8 => mat.ra.x(),
10782                                    }, @bitCast(std.math.shr(u64, imm, 8 * offset))),
10783                                    .i64 => |imm| switch (size) {
10784                                        else => unreachable,
10785                                        1...4 => try isel.movImmediate(mat.ra.w(), @as(u32, @bitCast(@as(i32, @truncate(std.math.shr(i64, imm, 8 * offset)))))),
10786                                        5...8 => try isel.movImmediate(mat.ra.x(), @bitCast(std.math.shr(i64, imm, 8 * offset))),
10787                                    },
10788                                    .big_int => |big_int| {
10789                                        assert(size == 8);
10790                                        var imm: u64 = 0;
10791                                        const limb_bits = @bitSizeOf(std.math.big.Limb);
10792                                        const limbs = @divExact(64, limb_bits);
10793                                        var limb_index: usize = @intCast(@divExact(offset, @divExact(limb_bits, 8)) + limbs);
10794                                        for (0..limbs) |_| {
10795                                            limb_index -= 1;
10796                                            if (limb_index >= big_int.limbs.len) continue;
10797                                            if (limb_bits < 64) imm <<= limb_bits;
10798                                            imm |= big_int.limbs[limb_index];
10799                                        }
10800                                        if (!big_int.positive) {
10801                                            limb_index = @min(limb_index, big_int.limbs.len);
10802                                            imm = while (limb_index > 0) {
10803                                                limb_index -= 1;
10804                                                if (big_int.limbs[limb_index] != 0) break ~imm;
10805                                            } else -%imm;
10806                                        }
10807                                        try isel.movImmediate(mat.ra.x(), imm);
10808                                    },
10809                                    .lazy_align => |ty| continue :storage .{
10810                                        .u64 = ZigType.fromInterned(ty).abiAlignment(zcu).toByteUnits().?,
10811                                    },
10812                                    .lazy_size => |ty| continue :storage .{
10813                                        .u64 = ZigType.fromInterned(ty).abiSize(zcu),
10814                                    },
10815                                },
10816                                .err => |err| continue :constant_key .{ .int = .{
10817                                    .ty = err.ty,
10818                                    .storage = .{ .u64 = ip.getErrorValueIfExists(err.name).? },
10819                                } },
10820                                .error_union => |error_union| {
10821                                    const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
10822                                    const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
10823                                    const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
10824                                    const error_set_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
10825                                    const error_set_size = error_set_ty.abiSize(zcu);
10826                                    if (offset >= error_set_offset and offset + size <= error_set_offset + error_set_size) {
10827                                        offset -= error_set_offset;
10828                                        continue :constant_key switch (error_union.val) {
10829                                            .err_name => |err_name| .{ .err = .{
10830                                                .ty = error_union_type.error_set_type,
10831                                                .name = err_name,
10832                                            } },
10833                                            .payload => .{ .int = .{
10834                                                .ty = error_union_type.error_set_type,
10835                                                .storage = .{ .u64 = 0 },
10836                                            } },
10837                                        };
10838                                    }
10839                                    const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
10840                                    const payload_size = payload_ty.abiSize(zcu);
10841                                    if (offset >= payload_offset and offset + size <= payload_offset + payload_size) {
10842                                        offset -= payload_offset;
10843                                        switch (error_union.val) {
10844                                            .err_name => continue :constant_key .{ .undef = error_union_type.payload_type },
10845                                            .payload => |payload| {
10846                                                constant = payload;
10847                                                constant_key = ip.indexToKey(constant);
10848                                                continue :constant_key constant_key;
10849                                            },
10850                                        }
10851                                    }
10852                                },
10853                                .enum_tag => |enum_tag| continue :constant_key .{ .int = ip.indexToKey(enum_tag.int).int },
10854                                .float => |float| storage: switch (float.storage) {
10855                                    .f16 => |imm| {
10856                                        if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10857                                            .ty = .u16_type,
10858                                            .storage = .{ .u64 = @as(u16, @bitCast(imm)) },
10859                                        } };
10860                                        const feat_fp16 = isel.target.cpu.has(.aarch64, .fullfp16);
10861                                        if (feat_fp16) {
10862                                            const Repr = std.math.FloatRepr(f16);
10863                                            const repr: Repr = @bitCast(imm);
10864                                            if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10865                                                .denormal, .infinite => false,
10866                                                else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10867                                            }) break :free try isel.emit(.fmov(mat.ra.h(), .{ .immediate = imm }));
10868                                        }
10869                                        const bits: u16 = @bitCast(imm);
10870                                        if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10871                                        if (bits & std.math.maxInt(u8) == 0) break :free try isel.emit(.movi(
10872                                            mat.ra.@"4h"(),
10873                                            @intCast(@shrExact(bits, 8)),
10874                                            .{ .lsl = 8 },
10875                                        ));
10876                                        const temp_ra = try isel.allocIntReg();
10877                                        defer isel.freeReg(temp_ra);
10878                                        try isel.emit(.fmov(if (feat_fp16) mat.ra.h() else mat.ra.s(), .{ .register = temp_ra.w() }));
10879                                        break :free try isel.movImmediate(temp_ra.w(), bits);
10880                                    },
10881                                    .f32 => |imm| {
10882                                        if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10883                                            .ty = .u32_type,
10884                                            .storage = .{ .u64 = @as(u32, @bitCast(imm)) },
10885                                        } };
10886                                        const Repr = std.math.FloatRepr(f32);
10887                                        const repr: Repr = @bitCast(imm);
10888                                        if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10889                                            .denormal, .infinite => false,
10890                                            else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10891                                        }) break :free try isel.emit(.fmov(mat.ra.s(), .{ .immediate = @floatCast(imm) }));
10892                                        const bits: u32 = @bitCast(imm);
10893                                        if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10894                                        if (bits & std.math.maxInt(u24) == 0) break :free try isel.emit(.movi(
10895                                            mat.ra.@"2s"(),
10896                                            @intCast(@shrExact(bits, 24)),
10897                                            .{ .lsl = 24 },
10898                                        ));
10899                                        const temp_ra = try isel.allocIntReg();
10900                                        defer isel.freeReg(temp_ra);
10901                                        try isel.emit(.fmov(mat.ra.s(), .{ .register = temp_ra.w() }));
10902                                        break :free try isel.movImmediate(temp_ra.w(), bits);
10903                                    },
10904                                    .f64 => |imm| {
10905                                        if (!mat.ra.isVector()) continue :constant_key .{ .int = .{
10906                                            .ty = .u64_type,
10907                                            .storage = .{ .u64 = @as(u64, @bitCast(imm)) },
10908                                        } };
10909                                        const Repr = std.math.FloatRepr(f64);
10910                                        const repr: Repr = @bitCast(imm);
10911                                        if (repr.mantissa & std.math.maxInt(Repr.Mantissa) >> 5 == 0 and switch (repr.exponent) {
10912                                            .denormal, .infinite => false,
10913                                            else => std.math.cast(i3, repr.exponent.unbias() - 1) != null,
10914                                        }) break :free try isel.emit(.fmov(mat.ra.d(), .{ .immediate = @floatCast(imm) }));
10915                                        const bits: u64 = @bitCast(imm);
10916                                        if (bits == 0) break :free try isel.emit(.movi(mat.ra.d(), 0b00000000, .replicate));
10917                                        const temp_ra = try isel.allocIntReg();
10918                                        defer isel.freeReg(temp_ra);
10919                                        try isel.emit(.fmov(mat.ra.d(), .{ .register = temp_ra.x() }));
10920                                        break :free try isel.movImmediate(temp_ra.x(), bits);
10921                                    },
10922                                    .f80 => |imm| break :free try isel.movImmediate(
10923                                        mat.ra.x(),
10924                                        @truncate(std.math.shr(u80, @bitCast(imm), 8 * offset)),
10925                                    ),
10926                                    .f128 => |imm| switch (ZigType.fromInterned(float.ty).floatBits(isel.target)) {
10927                                        else => unreachable,
10928                                        16 => continue :storage .{ .f16 = @floatCast(imm) },
10929                                        32 => continue :storage .{ .f32 = @floatCast(imm) },
10930                                        64 => continue :storage .{ .f64 = @floatCast(imm) },
10931                                        128 => {
10932                                            const bits: u128 = @bitCast(imm);
10933                                            const hi64: u64 = @intCast(bits >> 64);
10934                                            const lo64: u64 = @truncate(bits >> 0);
10935                                            const temp_ra = try isel.allocIntReg();
10936                                            defer isel.freeReg(temp_ra);
10937                                            switch (hi64) {
10938                                                0 => {},
10939                                                else => {
10940                                                    try isel.emit(.fmov(mat.ra.@"d[]"(1), .{ .register = temp_ra.x() }));
10941                                                    try isel.movImmediate(temp_ra.x(), hi64);
10942                                                },
10943                                            }
10944                                            break :free switch (lo64) {
10945                                                0 => try isel.emit(.movi(switch (hi64) {
10946                                                    else => mat.ra.d(),
10947                                                    0 => mat.ra.@"2d"(),
10948                                                }, 0b00000000, .replicate)),
10949                                                else => {
10950                                                    try isel.emit(.fmov(mat.ra.d(), .{ .register = temp_ra.x() }));
10951                                                    try isel.movImmediate(temp_ra.x(), lo64);
10952                                                },
10953                                            };
10954                                        },
10955                                    },
10956                                },
10957                                .ptr => |ptr| {
10958                                    assert(offset == 0 and size == 8);
10959                                    break :free switch (ptr.base_addr) {
10960                                        .nav => |nav| if (ZigType.fromInterned(ip.getNav(nav).typeOf(ip)).isFnOrHasRuntimeBits(zcu)) switch (true) {
10961                                            false => {
10962                                                try isel.nav_relocs.append(zcu.gpa, .{
10963                                                    .nav = nav,
10964                                                    .reloc = .{
10965                                                        .label = @intCast(isel.instructions.items.len),
10966                                                        .addend = ptr.byte_offset,
10967                                                    },
10968                                                });
10969                                                try isel.emit(.adr(mat.ra.x(), 0));
10970                                            },
10971                                            true => {
10972                                                try isel.nav_relocs.append(zcu.gpa, .{
10973                                                    .nav = nav,
10974                                                    .reloc = .{
10975                                                        .label = @intCast(isel.instructions.items.len),
10976                                                        .addend = ptr.byte_offset,
10977                                                    },
10978                                                });
10979                                                if (ip.getNav(nav).getExtern(ip)) |_|
10980                                                    try isel.emit(.ldr(mat.ra.x(), .{ .unsigned_offset = .{ .base = mat.ra.x(), .offset = 0 } }))
10981                                                else
10982                                                    try isel.emit(.add(mat.ra.x(), mat.ra.x(), .{ .immediate = 0 }));
10983                                                try isel.nav_relocs.append(zcu.gpa, .{
10984                                                    .nav = nav,
10985                                                    .reloc = .{
10986                                                        .label = @intCast(isel.instructions.items.len),
10987                                                        .addend = ptr.byte_offset,
10988                                                    },
10989                                                });
10990                                                try isel.emit(.adrp(mat.ra.x(), 0));
10991                                            },
10992                                        } else continue :constant_key .{ .int = .{
10993                                            .ty = .usize_type,
10994                                            .storage = .{ .u64 = isel.pt.navAlignment(nav).forward(0xaaaaaaaaaaaaaaaa) },
10995                                        } },
10996                                        .uav => |uav| if (ZigType.fromInterned(ip.typeOf(uav.val)).isFnOrHasRuntimeBits(zcu)) switch (true) {
10997                                            false => {
10998                                                try isel.uav_relocs.append(zcu.gpa, .{
10999                                                    .uav = uav,
11000                                                    .reloc = .{
11001                                                        .label = @intCast(isel.instructions.items.len),
11002                                                        .addend = ptr.byte_offset,
11003                                                    },
11004                                                });
11005                                                try isel.emit(.adr(mat.ra.x(), 0));
11006                                            },
11007                                            true => {
11008                                                try isel.uav_relocs.append(zcu.gpa, .{
11009                                                    .uav = uav,
11010                                                    .reloc = .{
11011                                                        .label = @intCast(isel.instructions.items.len),
11012                                                        .addend = ptr.byte_offset,
11013                                                    },
11014                                                });
11015                                                try isel.emit(.add(mat.ra.x(), mat.ra.x(), .{ .immediate = 0 }));
11016                                                try isel.uav_relocs.append(zcu.gpa, .{
11017                                                    .uav = uav,
11018                                                    .reloc = .{
11019                                                        .label = @intCast(isel.instructions.items.len),
11020                                                        .addend = ptr.byte_offset,
11021                                                    },
11022                                                });
11023                                                try isel.emit(.adrp(mat.ra.x(), 0));
11024                                            },
11025                                        } else continue :constant_key .{ .int = .{
11026                                            .ty = .usize_type,
11027                                            .storage = .{ .u64 = ZigType.fromInterned(uav.orig_ty).ptrAlignment(zcu).forward(0xaaaaaaaaaaaaaaaa) },
11028                                        } },
11029                                        .int => continue :constant_key .{ .int = .{
11030                                            .ty = .usize_type,
11031                                            .storage = .{ .u64 = ptr.byte_offset },
11032                                        } },
11033                                        .eu_payload => |base| {
11034                                            var base_ptr = ip.indexToKey(base).ptr;
11035                                            const eu_ty = ip.indexToKey(base_ptr.ty).ptr_type.child;
11036                                            const payload_ty = ip.indexToKey(eu_ty).error_union_type.payload_type;
11037                                            base_ptr.byte_offset += codegen.errUnionPayloadOffset(.fromInterned(payload_ty), zcu) + ptr.byte_offset;
11038                                            continue :constant_key .{ .ptr = base_ptr };
11039                                        },
11040                                        .opt_payload => |base| {
11041                                            var base_ptr = ip.indexToKey(base).ptr;
11042                                            base_ptr.byte_offset += ptr.byte_offset;
11043                                            continue :constant_key .{ .ptr = base_ptr };
11044                                        },
11045                                        .field => |field| {
11046                                            var base_ptr = ip.indexToKey(field.base).ptr;
11047                                            const agg_ty: ZigType = .fromInterned(ip.indexToKey(base_ptr.ty).ptr_type.child);
11048                                            base_ptr.byte_offset += agg_ty.structFieldOffset(@intCast(field.index), zcu) + ptr.byte_offset;
11049                                            continue :constant_key .{ .ptr = base_ptr };
11050                                        },
11051                                        .comptime_alloc, .comptime_field, .arr_elem => unreachable,
11052                                    };
11053                                },
11054                                .slice => |slice| switch (offset) {
11055                                    0 => continue :constant_key switch (ip.indexToKey(slice.ptr)) {
11056                                        else => unreachable,
11057                                        .undef => |undef| .{ .undef = undef },
11058                                        .ptr => |ptr| .{ .ptr = ptr },
11059                                    },
11060                                    else => {
11061                                        assert(offset == @divExact(isel.target.ptrBitWidth(), 8));
11062                                        offset = 0;
11063                                        continue :constant_key .{ .int = ip.indexToKey(slice.len).int };
11064                                    },
11065                                },
11066                                .opt => |opt| {
11067                                    const child_ty = ip.indexToKey(opt.ty).opt_type;
11068                                    const child_size = ZigType.fromInterned(child_ty).abiSize(zcu);
11069                                    if (offset == child_size and size == 1) {
11070                                        offset = 0;
11071                                        continue :constant_key .{ .simple_value = switch (opt.val) {
11072                                            .none => .false,
11073                                            else => .true,
11074                                        } };
11075                                    }
11076                                    const opt_ty: ZigType = .fromInterned(opt.ty);
11077                                    if (offset + size <= child_size) continue :constant_key switch (opt.val) {
11078                                        .none => if (opt_ty.optionalReprIsPayload(zcu)) .{ .int = .{
11079                                            .ty = opt.ty,
11080                                            .storage = .{ .u64 = 0 },
11081                                        } } else .{ .undef = child_ty },
11082                                        else => |child| {
11083                                            constant = child;
11084                                            constant_key = ip.indexToKey(constant);
11085                                            continue :constant_key constant_key;
11086                                        },
11087                                    };
11088                                },
11089                                .aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
11090                                    else => unreachable,
11091                                    .array_type => |array_type| {
11092                                        const elem_size = ZigType.fromInterned(array_type.child).abiSize(zcu);
11093                                        const elem_offset = @mod(offset, elem_size);
11094                                        if (size <= elem_size - elem_offset) {
11095                                            defer offset = elem_offset;
11096                                            continue :constant_key switch (aggregate.storage) {
11097                                                .bytes => |bytes| .{ .int = .{ .ty = .u8_type, .storage = .{
11098                                                    .u64 = bytes.toSlice(array_type.lenIncludingSentinel(), ip)[@intCast(@divFloor(offset, elem_size))],
11099                                                } } },
11100                                                .elems => |elems| {
11101                                                    constant = elems[@intCast(@divFloor(offset, elem_size))];
11102                                                    constant_key = ip.indexToKey(constant);
11103                                                    continue :constant_key constant_key;
11104                                                },
11105                                                .repeated_elem => |repeated_elem| {
11106                                                    constant = repeated_elem;
11107                                                    constant_key = ip.indexToKey(constant);
11108                                                    continue :constant_key constant_key;
11109                                                },
11110                                            };
11111                                        }
11112                                    },
11113                                    .vector_type => {},
11114                                    .struct_type => {
11115                                        const loaded_struct = ip.loadStructType(aggregate.ty);
11116                                        switch (loaded_struct.layout) {
11117                                            .auto => {
11118                                                var field_offset: u64 = 0;
11119                                                var field_it = loaded_struct.iterateRuntimeOrder(ip);
11120                                                while (field_it.next()) |field_index| {
11121                                                    if (loaded_struct.fieldIsComptime(ip, field_index)) continue;
11122                                                    const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
11123                                                    field_offset = field_ty.structFieldAlignment(
11124                                                        loaded_struct.fieldAlign(ip, field_index),
11125                                                        loaded_struct.layout,
11126                                                        zcu,
11127                                                    ).forward(field_offset);
11128                                                    const field_size = field_ty.abiSize(zcu);
11129                                                    if (offset >= field_offset and offset + size <= field_offset + field_size) {
11130                                                        offset -= field_offset;
11131                                                        constant = switch (aggregate.storage) {
11132                                                            .bytes => unreachable,
11133                                                            .elems => |elems| elems[field_index],
11134                                                            .repeated_elem => |repeated_elem| repeated_elem,
11135                                                        };
11136                                                        constant_key = ip.indexToKey(constant);
11137                                                        continue :constant_key constant_key;
11138                                                    }
11139                                                    field_offset += field_size;
11140                                                }
11141                                            },
11142                                            .@"extern", .@"packed" => {},
11143                                        }
11144                                    },
11145                                    .tuple_type => |tuple_type| {
11146                                        var field_offset: u64 = 0;
11147                                        for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
11148                                            if (field_value != .none) continue;
11149                                            const field_ty: ZigType = .fromInterned(field_type);
11150                                            field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
11151                                            const field_size = field_ty.abiSize(zcu);
11152                                            if (offset >= field_offset and offset + size <= field_offset + field_size) {
11153                                                offset -= field_offset;
11154                                                constant = switch (aggregate.storage) {
11155                                                    .bytes => unreachable,
11156                                                    .elems => |elems| elems[field_index],
11157                                                    .repeated_elem => |repeated_elem| repeated_elem,
11158                                                };
11159                                                constant_key = ip.indexToKey(constant);
11160                                                continue :constant_key constant_key;
11161                                            }
11162                                            field_offset += field_size;
11163                                        }
11164                                    },
11165                                },
11166                                .un => |un| {
11167                                    const loaded_union = ip.loadUnionType(un.ty);
11168                                    const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
11169                                    if (loaded_union.hasTag(ip)) {
11170                                        const tag_offset = union_layout.tagOffset();
11171                                        if (offset >= tag_offset and offset + size <= tag_offset + union_layout.tag_size) {
11172                                            offset -= tag_offset;
11173                                            continue :constant_key switch (ip.indexToKey(un.tag)) {
11174                                                else => unreachable,
11175                                                .int => |int| .{ .int = int },
11176                                                .enum_tag => |enum_tag| .{ .enum_tag = enum_tag },
11177                                            };
11178                                        }
11179                                    }
11180                                    const payload_offset = union_layout.payloadOffset();
11181                                    if (offset >= payload_offset and offset + size <= payload_offset + union_layout.payload_size) {
11182                                        offset -= payload_offset;
11183                                        constant = un.val;
11184                                        constant_key = ip.indexToKey(constant);
11185                                        continue :constant_key constant_key;
11186                                    }
11187                                },
11188                                else => {},
11189                            }
11190                            var buffer: [16]u8 = @splat(0);
11191                            if (ZigType.fromInterned(constant_key.typeOf()).abiSize(zcu) <= buffer.len and
11192                                try isel.writeToMemory(.fromInterned(constant), &buffer))
11193                            {
11194                                constant_key = if (mat.ra.isVector()) .{ .float = switch (size) {
11195                                    else => unreachable,
11196                                    2 => .{ .ty = .f16_type, .storage = .{ .f16 = @bitCast(std.mem.readInt(
11197                                        u16,
11198                                        buffer[@intCast(offset)..][0..2],
11199                                        isel.target.cpu.arch.endian(),
11200                                    )) } },
11201                                    4 => .{ .ty = .f32_type, .storage = .{ .f32 = @bitCast(std.mem.readInt(
11202                                        u32,
11203                                        buffer[@intCast(offset)..][0..4],
11204                                        isel.target.cpu.arch.endian(),
11205                                    )) } },
11206                                    8 => .{ .ty = .f64_type, .storage = .{ .f64 = @bitCast(std.mem.readInt(
11207                                        u64,
11208                                        buffer[@intCast(offset)..][0..8],
11209                                        isel.target.cpu.arch.endian(),
11210                                    )) } },
11211                                    16 => .{ .ty = .f128_type, .storage = .{ .f128 = @bitCast(std.mem.readInt(
11212                                        u128,
11213                                        buffer[@intCast(offset)..][0..16],
11214                                        isel.target.cpu.arch.endian(),
11215                                    )) } },
11216                                } } else .{ .int = .{
11217                                    .ty = .u64_type,
11218                                    .storage = .{ .u64 = switch (size) {
11219                                        else => unreachable,
11220                                        inline 1...8 => |ct_size| std.mem.readInt(
11221                                            @Int(.unsigned, 8 * ct_size),
11222                                            buffer[@intCast(offset)..][0..ct_size],
11223                                            isel.target.cpu.arch.endian(),
11224                                        ),
11225                                    } },
11226                                } };
11227                                offset = 0;
11228                                continue;
11229                            }
11230                            return isel.fail("unsupported value <{f}, {f}>", .{
11231                                isel.fmtType(.fromInterned(constant_key.typeOf())),
11232                                isel.fmtConstant(.fromInterned(constant)),
11233                            });
11234                        }
11235                    },
11236                }
11237            }
11238            live_vi.* = .free;
11239        }
11240    };
11241};
11242fn initValue(isel: *Select, ty: ZigType) Value.Index {
11243    const zcu = isel.pt.zcu;
11244    return isel.initValueAdvanced(ty.abiAlignment(zcu), 0, ty.abiSize(zcu));
11245}
11246fn initValueAdvanced(
11247    isel: *Select,
11248    parent_alignment: InternPool.Alignment,
11249    offset_from_parent: u64,
11250    size: u64,
11251) Value.Index {
11252    defer isel.values.addOneAssumeCapacity().* = .{
11253        .refs = 0,
11254        .flags = .{
11255            .alignment = .fromLog2Units(@min(parent_alignment.toLog2Units(), @ctz(offset_from_parent))),
11256            .parent_tag = .unallocated,
11257            .location_tag = if (size > 16) .large else .small,
11258            .parts_len_minus_one = 0,
11259        },
11260        .offset_from_parent = offset_from_parent,
11261        .parent_payload = .{ .unallocated = {} },
11262        .location_payload = if (size > 16) .{ .large = .{
11263            .size = size,
11264        } } else .{ .small = .{
11265            .size = @intCast(size),
11266            .signedness = .unsigned,
11267            .is_vector = false,
11268            .hint = .zr,
11269            .register = .zr,
11270        } },
11271        .parts = undefined,
11272    };
11273    return @enumFromInt(isel.values.items.len);
11274}
11275pub fn dumpValues(isel: *Select, which: enum { only_referenced, all }) void {
11276    errdefer |err| @panic(@errorName(err));
11277    const stderr, _ = std.debug.lockStderrWriter(&.{});
11278    defer std.debug.unlockStderrWriter();
11279
11280    const zcu = isel.pt.zcu;
11281    const gpa = zcu.gpa;
11282    const ip = &zcu.intern_pool;
11283    const nav = ip.getNav(isel.nav_index);
11284
11285    var reverse_live_values: std.AutoArrayHashMapUnmanaged(Value.Index, std.ArrayList(Air.Inst.Index)) = .empty;
11286    defer {
11287        for (reverse_live_values.values()) |*list| list.deinit(gpa);
11288        reverse_live_values.deinit(gpa);
11289    }
11290    {
11291        try reverse_live_values.ensureTotalCapacity(gpa, isel.live_values.count());
11292        var live_val_it = isel.live_values.iterator();
11293        while (live_val_it.next()) |live_val_entry| switch (live_val_entry.value_ptr.*) {
11294            _ => {
11295                const gop = reverse_live_values.getOrPutAssumeCapacity(live_val_entry.value_ptr.*);
11296                if (!gop.found_existing) gop.value_ptr.* = .empty;
11297                try gop.value_ptr.append(gpa, live_val_entry.key_ptr.*);
11298            },
11299            .allocating, .free => unreachable,
11300        };
11301    }
11302
11303    var reverse_live_registers: std.AutoHashMapUnmanaged(Value.Index, Register.Alias) = .empty;
11304    defer reverse_live_registers.deinit(gpa);
11305    {
11306        try reverse_live_registers.ensureTotalCapacity(gpa, @typeInfo(Register.Alias).@"enum".fields.len);
11307        var live_reg_it = isel.live_registers.iterator();
11308        while (live_reg_it.next()) |live_reg_entry| switch (live_reg_entry.value.*) {
11309            _ => reverse_live_registers.putAssumeCapacityNoClobber(live_reg_entry.value.*, live_reg_entry.key),
11310            .allocating, .free => {},
11311        };
11312    }
11313
11314    var roots: std.AutoArrayHashMapUnmanaged(Value.Index, u32) = .empty;
11315    defer roots.deinit(gpa);
11316    {
11317        try roots.ensureTotalCapacity(gpa, isel.values.items.len);
11318        var vi: Value.Index = @enumFromInt(isel.values.items.len);
11319        while (@intFromEnum(vi) > 0) {
11320            vi = @enumFromInt(@intFromEnum(vi) - 1);
11321            if (which == .only_referenced and vi.get(isel).refs == 0) continue;
11322            while (true) switch (vi.parent(isel)) {
11323                .unallocated, .stack_slot, .constant => break,
11324                .value => |parent_vi| vi = parent_vi,
11325                .address => |address_vi| break roots.putAssumeCapacity(address_vi, 0),
11326            };
11327            roots.putAssumeCapacity(vi, 0);
11328        }
11329    }
11330
11331    try stderr.print("# Begin {s} Value Dump: {f}:\n", .{ @typeName(Select), nav.fqn.fmt(ip) });
11332    while (roots.pop()) |root_entry| {
11333        const vi = root_entry.key;
11334        const value = vi.get(isel);
11335        try stderr.splatByteAll(' ', 2 * (@as(usize, 1) + root_entry.value));
11336        try stderr.print("${d}", .{@intFromEnum(vi)});
11337        {
11338            var first = true;
11339            if (reverse_live_values.get(vi)) |aiis| for (aiis.items) |aii| {
11340                if (aii == Block.main) {
11341                    try stderr.print("{s}%main", .{if (first) " <- " else ", "});
11342                } else {
11343                    try stderr.print("{s}%{d}", .{ if (first) " <- " else ", ", @intFromEnum(aii) });
11344                }
11345                first = false;
11346            };
11347            if (reverse_live_registers.get(vi)) |ra| {
11348                try stderr.print("{s}{t}", .{ if (first) " <- " else ", ", ra });
11349                first = false;
11350            }
11351        }
11352        try stderr.writeByte(':');
11353        switch (value.flags.parent_tag) {
11354            .unallocated => if (value.offset_from_parent != 0) try stderr.print(" +0x{x}", .{value.offset_from_parent}),
11355            .stack_slot => {
11356                try stderr.print(" [{t}, #{s}0x{x}", .{
11357                    value.parent_payload.stack_slot.base,
11358                    if (value.parent_payload.stack_slot.offset < 0) "-" else "",
11359                    @abs(value.parent_payload.stack_slot.offset),
11360                });
11361                if (value.offset_from_parent != 0) try stderr.print("+0x{x}", .{value.offset_from_parent});
11362                try stderr.writeByte(']');
11363            },
11364            .value => try stderr.print(" ${d}+0x{x}", .{ @intFromEnum(value.parent_payload.value), value.offset_from_parent }),
11365            .address => try stderr.print(" ${d}[0x{x}]", .{ @intFromEnum(value.parent_payload.address), value.offset_from_parent }),
11366            .constant => try stderr.print(" <{f}, {f}>", .{
11367                isel.fmtType(value.parent_payload.constant.typeOf(zcu)),
11368                isel.fmtConstant(value.parent_payload.constant),
11369            }),
11370        }
11371        try stderr.print(" align({t})", .{value.flags.alignment});
11372        switch (value.flags.location_tag) {
11373            .large => try stderr.print(" size=0x{x} large", .{value.location_payload.large.size}),
11374            .small => {
11375                const loc = value.location_payload.small;
11376                try stderr.print(" size=0x{x}", .{loc.size});
11377                switch (loc.signedness) {
11378                    .unsigned => {},
11379                    .signed => try stderr.writeAll(" signed"),
11380                }
11381                if (loc.hint != .zr) try stderr.print(" hint={t}", .{loc.hint});
11382                if (loc.register != .zr) try stderr.print(" loc={t}", .{loc.register});
11383            },
11384        }
11385        try stderr.print(" refs={d}\n", .{value.refs});
11386
11387        var part_index = value.flags.parts_len_minus_one;
11388        if (part_index > 0) while (true) : (part_index -= 1) {
11389            roots.putAssumeCapacityNoClobber(
11390                @enumFromInt(@intFromEnum(value.parts) + part_index),
11391                root_entry.value + 1,
11392            );
11393            if (part_index == 0) break;
11394        };
11395    }
11396    try stderr.print("# End {s} Value Dump: {f}\n\n", .{ @typeName(Select), nav.fqn.fmt(ip) });
11397}
11398
11399fn hasRepeatedByteRepr(isel: *Select, constant: Constant) error{OutOfMemory}!?u8 {
11400    const zcu = isel.pt.zcu;
11401    const ty = constant.typeOf(zcu);
11402    const abi_size = std.math.cast(usize, ty.abiSize(zcu)) orelse return null;
11403    const byte_buffer = try zcu.gpa.alloc(u8, abi_size);
11404    defer zcu.gpa.free(byte_buffer);
11405    return if (try isel.writeToMemory(constant, byte_buffer) and
11406        std.mem.allEqual(u8, byte_buffer[1..], byte_buffer[0])) byte_buffer[0] else null;
11407}
11408
11409fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMemory}!bool {
11410    const zcu = isel.pt.zcu;
11411    const ip = &zcu.intern_pool;
11412    if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11413    constant.writeToMemory(isel.pt, buffer) catch |err| switch (err) {
11414        error.OutOfMemory => return error.OutOfMemory,
11415        error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
11416    };
11417    return true;
11418}
11419fn writeKeyToMemory(isel: *Select, constant_key: InternPool.Key, buffer: []u8) error{OutOfMemory}!bool {
11420    const zcu = isel.pt.zcu;
11421    const ip = &zcu.intern_pool;
11422    switch (constant_key) {
11423        .int_type,
11424        .ptr_type,
11425        .array_type,
11426        .vector_type,
11427        .opt_type,
11428        .anyframe_type,
11429        .error_union_type,
11430        .simple_type,
11431        .struct_type,
11432        .tuple_type,
11433        .union_type,
11434        .opaque_type,
11435        .enum_type,
11436        .func_type,
11437        .error_set_type,
11438        .inferred_error_set_type,
11439
11440        .enum_literal,
11441        .empty_enum_value,
11442        .memoized_call,
11443        => unreachable, // not a runtime value
11444        .err => |err| {
11445            const error_int = ip.getErrorValueIfExists(err.name).?;
11446            switch (buffer.len) {
11447                else => unreachable,
11448                inline 1...4 => |size| std.mem.writeInt(
11449                    @Int(.unsigned, 8 * size),
11450                    buffer[0..size],
11451                    @intCast(error_int),
11452                    isel.target.cpu.arch.endian(),
11453                ),
11454            }
11455        },
11456        .error_union => |error_union| {
11457            const error_union_type = ip.indexToKey(error_union.ty).error_union_type;
11458            const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
11459            const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
11460            const error_set = buffer[@intCast(codegen.errUnionErrorOffset(payload_ty, zcu))..][0..@intCast(error_set_ty.abiSize(zcu))];
11461            switch (error_union.val) {
11462                .err_name => |err_name| if (!try isel.writeKeyToMemory(.{ .err = .{
11463                    .ty = error_set_ty.toIntern(),
11464                    .name = err_name,
11465                } }, error_set)) return false,
11466                .payload => |payload| {
11467                    if (!try isel.writeToMemory(
11468                        .fromInterned(payload),
11469                        buffer[@intCast(codegen.errUnionPayloadOffset(payload_ty, zcu))..][0..@intCast(payload_ty.abiSize(zcu))],
11470                    )) return false;
11471                    @memset(error_set, 0);
11472                },
11473            }
11474        },
11475        .opt => |opt| {
11476            const child_size: usize = @intCast(ZigType.fromInterned(ip.indexToKey(opt.ty).opt_type).abiSize(zcu));
11477            switch (opt.val) {
11478                .none => if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) {
11479                    buffer[child_size] = @intFromBool(false);
11480                } else @memset(buffer[0..child_size], 0x00),
11481                else => |child_constant| {
11482                    if (!try isel.writeToMemory(.fromInterned(child_constant), buffer[0..child_size])) return false;
11483                    if (!ZigType.fromInterned(opt.ty).optionalReprIsPayload(zcu)) buffer[child_size] = @intFromBool(true);
11484                },
11485            }
11486        },
11487        .aggregate => |aggregate| switch (ip.indexToKey(aggregate.ty)) {
11488            else => unreachable,
11489            .array_type => |array_type| {
11490                var elem_offset: usize = 0;
11491                const elem_size: usize = @intCast(ZigType.fromInterned(array_type.child).abiSize(zcu));
11492                const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
11493                switch (aggregate.storage) {
11494                    .bytes => |bytes| @memcpy(buffer[0..len_including_sentinel], bytes.toSlice(len_including_sentinel, ip)),
11495                    .elems => |elems| for (elems) |elem| {
11496                        if (!try isel.writeToMemory(.fromInterned(elem), buffer[elem_offset..][0..elem_size])) return false;
11497                        elem_offset += elem_size;
11498                    },
11499                    .repeated_elem => |repeated_elem| for (0..len_including_sentinel) |_| {
11500                        if (!try isel.writeToMemory(.fromInterned(repeated_elem), buffer[elem_offset..][0..elem_size])) return false;
11501                        elem_offset += elem_size;
11502                    },
11503                }
11504            },
11505            .vector_type => return false,
11506            .struct_type => {
11507                const loaded_struct = ip.loadStructType(aggregate.ty);
11508                switch (loaded_struct.layout) {
11509                    .auto => {
11510                        var field_offset: u64 = 0;
11511                        var field_it = loaded_struct.iterateRuntimeOrder(ip);
11512                        while (field_it.next()) |field_index| {
11513                            if (loaded_struct.fieldIsComptime(ip, field_index)) continue;
11514                            const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
11515                            field_offset = field_ty.structFieldAlignment(
11516                                loaded_struct.fieldAlign(ip, field_index),
11517                                loaded_struct.layout,
11518                                zcu,
11519                            ).forward(field_offset);
11520                            const field_size = field_ty.abiSize(zcu);
11521                            if (!try isel.writeToMemory(.fromInterned(switch (aggregate.storage) {
11522                                .bytes => unreachable,
11523                                .elems => |elems| elems[field_index],
11524                                .repeated_elem => |repeated_elem| repeated_elem,
11525                            }), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
11526                            field_offset += field_size;
11527                        }
11528                    },
11529                    .@"extern", .@"packed" => return false,
11530                }
11531            },
11532            .tuple_type => |tuple_type| {
11533                var field_offset: u64 = 0;
11534                for (tuple_type.types.get(ip), tuple_type.values.get(ip), 0..) |field_type, field_value, field_index| {
11535                    if (field_value != .none) continue;
11536                    const field_ty: ZigType = .fromInterned(field_type);
11537                    field_offset = field_ty.abiAlignment(zcu).forward(field_offset);
11538                    const field_size = field_ty.abiSize(zcu);
11539                    if (!try isel.writeToMemory(.fromInterned(switch (aggregate.storage) {
11540                        .bytes => unreachable,
11541                        .elems => |elems| elems[field_index],
11542                        .repeated_elem => |repeated_elem| repeated_elem,
11543                    }), buffer[@intCast(field_offset)..][0..@intCast(field_size)])) return false;
11544                    field_offset += field_size;
11545                }
11546            },
11547        },
11548        else => return false,
11549    }
11550    return true;
11551}
11552
11553const TryAllocRegResult = union(enum) {
11554    allocated: Register.Alias,
11555    fill_candidate: Register.Alias,
11556    out_of_registers,
11557};
11558
11559fn tryAllocIntReg(isel: *Select) TryAllocRegResult {
11560    var failed_result: TryAllocRegResult = .out_of_registers;
11561    var ra: Register.Alias = .r0;
11562    while (true) : (ra = @enumFromInt(@intFromEnum(ra) + 1)) {
11563        if (ra == .r18) continue; // The Platform Register
11564        if (ra == Register.Alias.fp) continue;
11565        const live_vi = isel.live_registers.getPtr(ra);
11566        switch (live_vi.*) {
11567            _ => switch (failed_result) {
11568                .allocated => unreachable,
11569                .fill_candidate => {},
11570                .out_of_registers => failed_result = .{ .fill_candidate = ra },
11571            },
11572            .allocating => {},
11573            .free => {
11574                live_vi.* = .allocating;
11575                isel.saved_registers.insert(ra);
11576                return .{ .allocated = ra };
11577            },
11578        }
11579        if (ra == Register.Alias.lr) return failed_result;
11580    }
11581}
11582
11583fn allocIntReg(isel: *Select) !Register.Alias {
11584    switch (isel.tryAllocIntReg()) {
11585        .allocated => |ra| return ra,
11586        .fill_candidate => |ra| {
11587            assert(try isel.fillMemory(ra));
11588            const live_vi = isel.live_registers.getPtr(ra);
11589            assert(live_vi.* == .free);
11590            live_vi.* = .allocating;
11591            return ra;
11592        },
11593        .out_of_registers => return isel.fail("ran out of registers", .{}),
11594    }
11595}
11596
11597fn tryAllocVecReg(isel: *Select) TryAllocRegResult {
11598    var failed_result: TryAllocRegResult = .out_of_registers;
11599    var ra: Register.Alias = .v0;
11600    while (true) : (ra = @enumFromInt(@intFromEnum(ra) + 1)) {
11601        const live_vi = isel.live_registers.getPtr(ra);
11602        switch (live_vi.*) {
11603            _ => switch (failed_result) {
11604                .allocated => unreachable,
11605                .fill_candidate => {},
11606                .out_of_registers => failed_result = .{ .fill_candidate = ra },
11607            },
11608            .allocating => {},
11609            .free => {
11610                live_vi.* = .allocating;
11611                isel.saved_registers.insert(ra);
11612                return .{ .allocated = ra };
11613            },
11614        }
11615        if (ra == Register.Alias.v31) return failed_result;
11616    }
11617}
11618
11619fn allocVecReg(isel: *Select) !Register.Alias {
11620    switch (isel.tryAllocVecReg()) {
11621        .allocated => |ra| return ra,
11622        .fill_candidate => |ra| {
11623            assert(try isel.fillMemory(ra));
11624            return ra;
11625        },
11626        .out_of_registers => return isel.fail("ran out of registers", .{}),
11627    }
11628}
11629
11630const RegLock = struct {
11631    ra: Register.Alias,
11632    const empty: RegLock = .{ .ra = .zr };
11633    fn unlock(lock: RegLock, isel: *Select) void {
11634        switch (lock.ra) {
11635            else => |ra| isel.freeReg(ra),
11636            .zr => {},
11637        }
11638    }
11639};
11640fn lockReg(isel: *Select, ra: Register.Alias) RegLock {
11641    assert(ra != .zr);
11642    const live_vi = isel.live_registers.getPtr(ra);
11643    assert(live_vi.* == .free);
11644    live_vi.* = .allocating;
11645    return .{ .ra = ra };
11646}
11647fn tryLockReg(isel: *Select, ra: Register.Alias) RegLock {
11648    assert(ra != .zr);
11649    const live_vi = isel.live_registers.getPtr(ra);
11650    switch (live_vi.*) {
11651        _ => unreachable,
11652        .allocating => return .{ .ra = .zr },
11653        .free => {
11654            live_vi.* = .allocating;
11655            return .{ .ra = ra };
11656        },
11657    }
11658}
11659
11660fn freeReg(isel: *Select, ra: Register.Alias) void {
11661    assert(ra != .zr);
11662    const live_vi = isel.live_registers.getPtr(ra);
11663    assert(live_vi.* == .allocating);
11664    live_vi.* = .free;
11665}
11666
11667fn use(isel: *Select, air_ref: Air.Inst.Ref) !Value.Index {
11668    const zcu = isel.pt.zcu;
11669    const ip = &zcu.intern_pool;
11670    try isel.values.ensureUnusedCapacity(zcu.gpa, 1);
11671    const vi, const ty = if (air_ref.toIndex()) |air_inst_index| vi_ty: {
11672        const live_gop = try isel.live_values.getOrPut(zcu.gpa, air_inst_index);
11673        if (live_gop.found_existing) return live_gop.value_ptr.*;
11674        const ty = isel.air.typeOf(air_ref, ip);
11675        const vi = isel.initValue(ty);
11676        tracking_log.debug("${d} <- %{d}", .{
11677            @intFromEnum(vi),
11678            @intFromEnum(air_inst_index),
11679        });
11680        live_gop.value_ptr.* = vi.ref(isel);
11681        break :vi_ty .{ vi, ty };
11682    } else vi_ty: {
11683        const constant: Constant = .fromInterned(air_ref.toInterned().?);
11684        const ty = constant.typeOf(zcu);
11685        const vi = isel.initValue(ty);
11686        tracking_log.debug("${d} <- <{f}, {f}>", .{
11687            @intFromEnum(vi),
11688            isel.fmtType(ty),
11689            isel.fmtConstant(constant),
11690        });
11691        vi.setParent(isel, .{ .constant = constant });
11692        break :vi_ty .{ vi, ty };
11693    };
11694    if (ty.isAbiInt(zcu)) {
11695        const int_info = ty.intInfo(zcu);
11696        if (int_info.bits <= 16) vi.setSignedness(isel, int_info.signedness);
11697    } else if (vi.size(isel) <= 16 and
11698        CallAbiIterator.homogeneousAggregateBaseType(zcu, ty.toIntern()) != null) vi.setIsVector(isel);
11699    return vi;
11700}
11701
11702fn fill(isel: *Select, dst_ra: Register.Alias) error{ OutOfMemory, CodegenFail }!bool {
11703    switch (dst_ra) {
11704        else => {},
11705        Register.Alias.fp, .zr, .sp, .pc, .fpcr, .fpsr, .ffr => return false,
11706    }
11707    const dst_live_vi = isel.live_registers.getPtr(dst_ra);
11708    const dst_vi = switch (dst_live_vi.*) {
11709        _ => |dst_vi| dst_vi,
11710        .allocating => return false,
11711        .free => return true,
11712    };
11713    const src_ra = src_ra: {
11714        if (dst_vi.hint(isel)) |hint_ra| {
11715            assert(dst_live_vi.* == dst_vi);
11716            dst_live_vi.* = .allocating;
11717            defer dst_live_vi.* = dst_vi;
11718            if (try isel.fill(hint_ra)) {
11719                isel.saved_registers.insert(hint_ra);
11720                break :src_ra hint_ra;
11721            }
11722        }
11723        switch (if (dst_vi.isVector(isel)) isel.tryAllocVecReg() else isel.tryAllocIntReg()) {
11724            .allocated => |ra| break :src_ra ra,
11725            .fill_candidate, .out_of_registers => return isel.fillMemory(dst_ra),
11726        }
11727    };
11728    try dst_vi.liveIn(isel, src_ra, comptime &.initFill(.free));
11729    const src_live_vi = isel.live_registers.getPtr(src_ra);
11730    assert(src_live_vi.* == .allocating);
11731    src_live_vi.* = dst_vi;
11732    return true;
11733}
11734
11735fn fillMemory(isel: *Select, dst_ra: Register.Alias) error{ OutOfMemory, CodegenFail }!bool {
11736    const dst_live_vi = isel.live_registers.getPtr(dst_ra);
11737    const dst_vi = switch (dst_live_vi.*) {
11738        _ => |dst_vi| dst_vi,
11739        .allocating => return false,
11740        .free => return true,
11741    };
11742    const dst_vi_ra = &dst_vi.get(isel).location_payload.small.register;
11743    assert(dst_vi_ra.* == dst_ra);
11744    const base_ra = if (dst_ra.isVector()) try isel.allocIntReg() else dst_ra;
11745    defer if (base_ra != dst_ra) isel.freeReg(base_ra);
11746    try isel.emit(switch (dst_vi.size(isel)) {
11747        else => unreachable,
11748        1 => if (dst_ra.isVector())
11749            .ldr(dst_ra.b(), .{ .base = base_ra.x() })
11750        else switch (dst_vi.signedness(isel)) {
11751            .signed => .ldrsb(dst_ra.w(), .{ .base = base_ra.x() }),
11752            .unsigned => .ldrb(dst_ra.w(), .{ .base = base_ra.x() }),
11753        },
11754        2 => if (dst_ra.isVector())
11755            .ldr(dst_ra.h(), .{ .base = base_ra.x() })
11756        else switch (dst_vi.signedness(isel)) {
11757            .signed => .ldrsh(dst_ra.w(), .{ .base = base_ra.x() }),
11758            .unsigned => .ldrh(dst_ra.w(), .{ .base = base_ra.x() }),
11759        },
11760        4 => .ldr(if (dst_ra.isVector()) dst_ra.s() else dst_ra.w(), .{ .base = base_ra.x() }),
11761        8 => .ldr(if (dst_ra.isVector()) dst_ra.d() else dst_ra.x(), .{ .base = base_ra.x() }),
11762        16 => .ldr(dst_ra.q(), .{ .base = base_ra.x() }),
11763    });
11764    dst_vi_ra.* = .zr;
11765    try dst_vi.address(isel, 0, base_ra);
11766    dst_live_vi.* = .free;
11767    return true;
11768}
11769
11770/// Merges possibly differing value tracking into a consistent state.
11771///
11772/// At a conditional branch, if a value is expected in the same register on both
11773/// paths, or only expected in a register on only one path, tracking is updated:
11774///
11775///     $0 -> r0 // final state is now consistent with both paths
11776///      b.cond else
11777///     then:
11778///     $0 -> r0 // updated if not already consistent with else
11779///      ...
11780///      b end
11781///     else:
11782///     $0 -> r0
11783///      ...
11784///     end:
11785///
11786/// At a conditional branch, if a value is expected in different registers on
11787/// each path, mov instructions are emitted:
11788///
11789///     $0 -> r0 // final state is now consistent with both paths
11790///      b.cond else
11791///     then:
11792///     $0 -> r0 // updated to be consistent with else
11793///      mov x1, x0 // emitted to merge the inconsistent states
11794///     $0 -> r1
11795///      ...
11796///      b end
11797///     else:
11798///     $0 -> r0
11799///      ...
11800///     end:
11801///
11802/// At a loop, a value that is expected in a register at the repeats is updated:
11803///
11804///     $0 -> r0 // final state is now consistent with all paths
11805///     loop:
11806///     $0 -> r0 // updated to be consistent with the repeats
11807///      ...
11808///     $0 -> r0
11809///      b.cond loop
11810///      ...
11811///     $0 -> r0
11812///      b loop
11813///
11814/// At a loop, a value that is expected in a register at the top is filled:
11815///
11816///     $0 -> [sp, #A] // final state is now consistent with all paths
11817///     loop:
11818///     $0 -> [sp, #A] // updated to be consistent with the repeats
11819///      ldr x0, [sp, #A] // emitted to merge the inconsistent states
11820///     $0 -> r0
11821///      ...
11822///     $0 -> [sp, #A]
11823///      b.cond loop
11824///      ...
11825///     $0 -> [sp, #A]
11826///      b loop
11827///
11828/// At a loop, if a value that is expected in different registers on each path,
11829/// mov instructions are emitted:
11830///
11831///     $0 -> r0 // final state is now consistent with all paths
11832///     loop:
11833///     $0 -> r0 // updated to be consistent with the repeats
11834///      mov x1, x0 // emitted to merge the inconsistent states
11835///     $0 -> r1
11836///      ...
11837///     $0 -> r0
11838///      b.cond loop
11839///      ...
11840///     $0 -> r0
11841///      b loop
11842fn merge(
11843    isel: *Select,
11844    expected_live_registers: *const LiveRegisters,
11845    comptime opts: struct { fill_extra: bool = false },
11846) !void {
11847    var live_reg_it = isel.live_registers.iterator();
11848    while (live_reg_it.next()) |live_reg_entry| {
11849        const ra = live_reg_entry.key;
11850        const actual_vi = live_reg_entry.value;
11851        const expected_vi = expected_live_registers.get(ra);
11852        switch (expected_vi) {
11853            else => switch (actual_vi.*) {
11854                _ => {},
11855                .allocating => unreachable,
11856                .free => actual_vi.* = .allocating,
11857            },
11858            .free => {},
11859        }
11860    }
11861    live_reg_it = isel.live_registers.iterator();
11862    while (live_reg_it.next()) |live_reg_entry| {
11863        const ra = live_reg_entry.key;
11864        const actual_vi = live_reg_entry.value;
11865        const expected_vi = expected_live_registers.get(ra);
11866        switch (expected_vi) {
11867            _ => {
11868                switch (actual_vi.*) {
11869                    _ => _ = if (opts.fill_extra) {
11870                        assert(try isel.fillMemory(ra));
11871                        assert(actual_vi.* == .free);
11872                    },
11873                    .allocating => actual_vi.* = .free,
11874                    .free => unreachable,
11875                }
11876                try expected_vi.liveIn(isel, ra, expected_live_registers);
11877            },
11878            .allocating => if (if (opts.fill_extra) try isel.fillMemory(ra) else try isel.fill(ra)) {
11879                assert(actual_vi.* == .free);
11880                actual_vi.* = .allocating;
11881            },
11882            .free => if (opts.fill_extra) assert(try isel.fillMemory(ra) and actual_vi.* == .free),
11883        }
11884    }
11885    live_reg_it = isel.live_registers.iterator();
11886    while (live_reg_it.next()) |live_reg_entry| {
11887        const ra = live_reg_entry.key;
11888        const actual_vi = live_reg_entry.value;
11889        const expected_vi = expected_live_registers.get(ra);
11890        switch (expected_vi) {
11891            _ => {
11892                assert(actual_vi.* == .allocating and expected_vi.register(isel) == ra);
11893                actual_vi.* = expected_vi;
11894            },
11895            .allocating => assert(actual_vi.* == .allocating),
11896            .free => if (opts.fill_extra) assert(actual_vi.* == .free),
11897        }
11898    }
11899}
11900
11901const call = struct {
11902    const param_reg: Value.Index = @enumFromInt(@intFromEnum(Value.Index.allocating) - 2);
11903    const callee_clobbered_reg: Value.Index = @enumFromInt(@intFromEnum(Value.Index.allocating) - 1);
11904    const caller_saved_regs: LiveRegisters = .init(.{
11905        .r0 = param_reg,
11906        .r1 = param_reg,
11907        .r2 = param_reg,
11908        .r3 = param_reg,
11909        .r4 = param_reg,
11910        .r5 = param_reg,
11911        .r6 = param_reg,
11912        .r7 = param_reg,
11913        .r8 = param_reg,
11914        .r9 = callee_clobbered_reg,
11915        .r10 = callee_clobbered_reg,
11916        .r11 = callee_clobbered_reg,
11917        .r12 = callee_clobbered_reg,
11918        .r13 = callee_clobbered_reg,
11919        .r14 = callee_clobbered_reg,
11920        .r15 = callee_clobbered_reg,
11921        .r16 = callee_clobbered_reg,
11922        .r17 = callee_clobbered_reg,
11923        .r18 = callee_clobbered_reg,
11924        .r19 = .free,
11925        .r20 = .free,
11926        .r21 = .free,
11927        .r22 = .free,
11928        .r23 = .free,
11929        .r24 = .free,
11930        .r25 = .free,
11931        .r26 = .free,
11932        .r27 = .free,
11933        .r28 = .free,
11934        .r29 = .free,
11935        .r30 = callee_clobbered_reg,
11936        .zr = .free,
11937        .sp = .free,
11938
11939        .pc = .free,
11940
11941        .v0 = param_reg,
11942        .v1 = param_reg,
11943        .v2 = param_reg,
11944        .v3 = param_reg,
11945        .v4 = param_reg,
11946        .v5 = param_reg,
11947        .v6 = param_reg,
11948        .v7 = param_reg,
11949        .v8 = .free,
11950        .v9 = .free,
11951        .v10 = .free,
11952        .v11 = .free,
11953        .v12 = .free,
11954        .v13 = .free,
11955        .v14 = .free,
11956        .v15 = .free,
11957        .v16 = callee_clobbered_reg,
11958        .v17 = callee_clobbered_reg,
11959        .v18 = callee_clobbered_reg,
11960        .v19 = callee_clobbered_reg,
11961        .v20 = callee_clobbered_reg,
11962        .v21 = callee_clobbered_reg,
11963        .v22 = callee_clobbered_reg,
11964        .v23 = callee_clobbered_reg,
11965        .v24 = callee_clobbered_reg,
11966        .v25 = callee_clobbered_reg,
11967        .v26 = callee_clobbered_reg,
11968        .v27 = callee_clobbered_reg,
11969        .v28 = callee_clobbered_reg,
11970        .v29 = callee_clobbered_reg,
11971        .v30 = callee_clobbered_reg,
11972        .v31 = callee_clobbered_reg,
11973
11974        .fpcr = .free,
11975        .fpsr = .free,
11976
11977        .p0 = callee_clobbered_reg,
11978        .p1 = callee_clobbered_reg,
11979        .p2 = callee_clobbered_reg,
11980        .p3 = callee_clobbered_reg,
11981        .p4 = callee_clobbered_reg,
11982        .p5 = callee_clobbered_reg,
11983        .p6 = callee_clobbered_reg,
11984        .p7 = callee_clobbered_reg,
11985        .p8 = callee_clobbered_reg,
11986        .p9 = callee_clobbered_reg,
11987        .p10 = callee_clobbered_reg,
11988        .p11 = callee_clobbered_reg,
11989        .p12 = callee_clobbered_reg,
11990        .p13 = callee_clobbered_reg,
11991        .p14 = callee_clobbered_reg,
11992        .p15 = callee_clobbered_reg,
11993
11994        .ffr = .free,
11995    });
11996    fn prepareReturn(isel: *Select) !void {
11997        var live_reg_it = isel.live_registers.iterator();
11998        while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
11999            else => unreachable,
12000            param_reg, callee_clobbered_reg => switch (live_reg_entry.value.*) {
12001                _ => {},
12002                .allocating => unreachable,
12003                .free => live_reg_entry.value.* = .allocating,
12004            },
12005            .free => {},
12006        };
12007    }
12008    fn returnFill(isel: *Select, ra: Register.Alias) !void {
12009        const live_vi = isel.live_registers.getPtr(ra);
12010        if (try isel.fill(ra)) {
12011            assert(live_vi.* == .free);
12012            live_vi.* = .allocating;
12013        }
12014        assert(live_vi.* == .allocating);
12015    }
12016    fn returnLiveIn(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
12017        try vi.defLiveIn(isel, ra, &caller_saved_regs);
12018    }
12019    fn finishReturn(isel: *Select) !void {
12020        var live_reg_it = isel.live_registers.iterator();
12021        while (live_reg_it.next()) |live_reg_entry| {
12022            switch (live_reg_entry.value.*) {
12023                _ => |live_vi| switch (live_vi.size(isel)) {
12024                    else => unreachable,
12025                    1, 2, 4, 8 => {},
12026                    16 => {
12027                        assert(try isel.fillMemory(live_reg_entry.key));
12028                        assert(live_reg_entry.value.* == .free);
12029                        switch (caller_saved_regs.get(live_reg_entry.key)) {
12030                            else => unreachable,
12031                            param_reg, callee_clobbered_reg => live_reg_entry.value.* = .allocating,
12032                            .free => {},
12033                        }
12034                        continue;
12035                    },
12036                },
12037                .allocating, .free => {},
12038            }
12039            switch (caller_saved_regs.get(live_reg_entry.key)) {
12040                else => unreachable,
12041                param_reg, callee_clobbered_reg => switch (live_reg_entry.value.*) {
12042                    _ => {
12043                        assert(try isel.fill(live_reg_entry.key));
12044                        assert(live_reg_entry.value.* == .free);
12045                        live_reg_entry.value.* = .allocating;
12046                    },
12047                    .allocating => {},
12048                    .free => unreachable,
12049                },
12050                .free => {},
12051            }
12052        }
12053    }
12054    fn prepareCallee(isel: *Select) !void {
12055        var live_reg_it = isel.live_registers.iterator();
12056        while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
12057            else => unreachable,
12058            param_reg => assert(live_reg_entry.value.* == .allocating),
12059            callee_clobbered_reg => isel.freeReg(live_reg_entry.key),
12060            .free => {},
12061        };
12062    }
12063    fn finishCallee(_: *Select) !void {}
12064    fn prepareParams(_: *Select) !void {}
12065    fn paramLiveOut(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
12066        isel.freeReg(ra);
12067        try vi.liveOut(isel, ra);
12068        const live_vi = isel.live_registers.getPtr(ra);
12069        if (live_vi.* == .free) live_vi.* = .allocating;
12070    }
12071    fn paramAddress(isel: *Select, vi: Value.Index, ra: Register.Alias) !void {
12072        isel.freeReg(ra);
12073        try vi.address(isel, 0, ra);
12074        const live_vi = isel.live_registers.getPtr(ra);
12075        if (live_vi.* == .free) live_vi.* = .allocating;
12076    }
12077    fn finishParams(isel: *Select) !void {
12078        var live_reg_it = isel.live_registers.iterator();
12079        while (live_reg_it.next()) |live_reg_entry| switch (caller_saved_regs.get(live_reg_entry.key)) {
12080            else => unreachable,
12081            param_reg => switch (live_reg_entry.value.*) {
12082                _ => {},
12083                .allocating => live_reg_entry.value.* = .free,
12084                .free => unreachable,
12085            },
12086            callee_clobbered_reg, .free => {},
12087        };
12088    }
12089};
12090
12091pub const CallAbiIterator = struct {
12092    /// Next General-purpose Register Number
12093    ngrn: Register.Alias,
12094    /// Next SIMD and Floating-point Register Number
12095    nsrn: Register.Alias,
12096    /// next stacked argument address
12097    nsaa: u24,
12098
12099    pub const ngrn_start: Register.Alias = .r0;
12100    pub const ngrn_end: Register.Alias = .r8;
12101    pub const nsrn_start: Register.Alias = .v0;
12102    pub const nsrn_end: Register.Alias = .v8;
12103    pub const nsaa_start: u42 = 0;
12104
12105    pub const init: CallAbiIterator = .{
12106        // A.1
12107        .ngrn = ngrn_start,
12108        // A.2
12109        .nsrn = nsrn_start,
12110        // A.3
12111        .nsaa = nsaa_start,
12112    };
12113
12114    pub fn param(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12115        const zcu = isel.pt.zcu;
12116        const ip = &zcu.intern_pool;
12117
12118        if (ty.isNoReturn(zcu) or !ty.hasRuntimeBitsIgnoreComptime(zcu)) return null;
12119        try isel.values.ensureUnusedCapacity(zcu.gpa, Value.max_parts);
12120        const wip_vi = isel.initValue(ty);
12121        type_key: switch (ip.indexToKey(ty.toIntern())) {
12122            else => return isel.fail("CallAbiIterator.param({f})", .{isel.fmtType(ty)}),
12123            .int_type => |int_type| switch (int_type.bits) {
12124                0 => unreachable,
12125                1...16 => {
12126                    wip_vi.setSignedness(isel, int_type.signedness);
12127                    // C.7
12128                    it.integer(isel, wip_vi);
12129                },
12130                // C.7
12131                17...64 => it.integer(isel, wip_vi),
12132                // C.9
12133                65...128 => it.integers(isel, wip_vi, @splat(@divExact(wip_vi.size(isel), 2))),
12134                else => it.indirect(isel, wip_vi),
12135            },
12136            .array_type => switch (wip_vi.size(isel)) {
12137                0 => unreachable,
12138                1...8 => it.integer(isel, wip_vi),
12139                9...16 => |size| it.integers(isel, wip_vi, .{ 8, size - 8 }),
12140                else => it.indirect(isel, wip_vi),
12141            },
12142            .ptr_type => |ptr_type| switch (ptr_type.flags.size) {
12143                .one, .many, .c => continue :type_key .{ .int_type = .{
12144                    .signedness = .unsigned,
12145                    .bits = 64,
12146                } },
12147                .slice => it.integers(isel, wip_vi, @splat(8)),
12148            },
12149            .opt_type => |child_type| if (ty.optionalReprIsPayload(zcu))
12150                continue :type_key ip.indexToKey(child_type)
12151            else switch (ZigType.fromInterned(child_type).abiSize(zcu)) {
12152                0 => continue :type_key .{ .simple_type = .bool },
12153                1...7 => it.integer(isel, wip_vi),
12154                8...15 => |child_size| it.integers(isel, wip_vi, .{ 8, child_size - 7 }),
12155                else => return isel.fail("CallAbiIterator.param({f})", .{isel.fmtType(ty)}),
12156            },
12157            .anyframe_type => unreachable,
12158            .error_union_type => |error_union_type| switch (wip_vi.size(isel)) {
12159                0 => unreachable,
12160                1...8 => it.integer(isel, wip_vi),
12161                9...16 => {
12162                    var sizes: [2]u64 = @splat(0);
12163                    const payload_ty: ZigType = .fromInterned(error_union_type.payload_type);
12164                    {
12165                        const error_set_ty: ZigType = .fromInterned(error_union_type.error_set_type);
12166                        const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
12167                        const end = offset % 8 + error_set_ty.abiSize(zcu);
12168                        const part_index: usize = @intCast(offset / 8);
12169                        sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12170                        if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12171                    }
12172                    {
12173                        const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
12174                        const end = offset % 8 + payload_ty.abiSize(zcu);
12175                        const part_index: usize = @intCast(offset / 8);
12176                        sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12177                        if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12178                    }
12179                    it.integers(isel, wip_vi, sizes);
12180                },
12181                else => it.indirect(isel, wip_vi),
12182            },
12183            .simple_type => |simple_type| switch (simple_type) {
12184                .f16, .f32, .f64, .f128, .c_longdouble => it.vector(isel, wip_vi),
12185                .f80 => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 80 } },
12186                .usize,
12187                .isize,
12188                .c_char,
12189                .c_short,
12190                .c_ushort,
12191                .c_int,
12192                .c_uint,
12193                .c_long,
12194                .c_ulong,
12195                .c_longlong,
12196                .c_ulonglong,
12197                => continue :type_key .{ .int_type = ty.intInfo(zcu) },
12198                // B.1
12199                .anyopaque => it.indirect(isel, wip_vi),
12200                .bool => continue :type_key .{ .int_type = .{ .signedness = .unsigned, .bits = 1 } },
12201                .anyerror => continue :type_key .{ .int_type = .{
12202                    .signedness = .unsigned,
12203                    .bits = zcu.errorSetBits(),
12204                } },
12205                .void,
12206                .type,
12207                .comptime_int,
12208                .comptime_float,
12209                .noreturn,
12210                .null,
12211                .undefined,
12212                .enum_literal,
12213                .adhoc_inferred_error_set,
12214                .generic_poison,
12215                => unreachable,
12216            },
12217            .struct_type => {
12218                const loaded_struct = ip.loadStructType(ty.toIntern());
12219                switch (loaded_struct.layout) {
12220                    .auto, .@"extern" => {},
12221                    .@"packed" => continue :type_key .{
12222                        .int_type = ip.indexToKey(loaded_struct.backingIntTypeUnordered(ip)).int_type,
12223                    },
12224                }
12225                const size = wip_vi.size(isel);
12226                if (size <= 16 * 4) homogeneous_aggregate: {
12227                    const fdt = homogeneousStructBaseType(zcu, &loaded_struct) orelse break :homogeneous_aggregate;
12228                    const parts_len = @shrExact(size, fdt.log2Size());
12229                    if (parts_len > 4) break :homogeneous_aggregate;
12230                    it.vectors(isel, wip_vi, fdt, @intCast(parts_len));
12231                    break :type_key;
12232                }
12233                switch (size) {
12234                    0 => unreachable,
12235                    1...8 => it.integer(isel, wip_vi),
12236                    9...16 => {
12237                        var part_offset: u64 = 0;
12238                        var part_sizes: [2]u64 = undefined;
12239                        var parts_len: Value.PartsLen = 0;
12240                        var next_field_end: u64 = 0;
12241                        var field_it = loaded_struct.iterateRuntimeOrder(ip);
12242                        while (part_offset < size) {
12243                            const field_end = next_field_end;
12244                            const next_field_begin = if (field_it.next()) |field_index| next_field_begin: {
12245                                const field_ty: ZigType = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
12246                                const next_field_begin = switch (loaded_struct.fieldAlign(ip, field_index)) {
12247                                    .none => field_ty.abiAlignment(zcu),
12248                                    else => |field_align| field_align,
12249                                }.forward(field_end);
12250                                next_field_end = next_field_begin + field_ty.abiSize(zcu);
12251                                break :next_field_begin next_field_begin;
12252                            } else std.mem.alignForward(u64, size, 8);
12253                            while (next_field_begin - part_offset >= 8) {
12254                                const part_size = field_end - part_offset;
12255                                part_sizes[parts_len] = part_size;
12256                                assert(part_offset + part_size <= size);
12257                                parts_len += 1;
12258                                part_offset = next_field_begin;
12259                            }
12260                        }
12261                        assert(parts_len == part_sizes.len);
12262                        it.integers(isel, wip_vi, part_sizes);
12263                    },
12264                    else => it.indirect(isel, wip_vi),
12265                }
12266            },
12267            .tuple_type => |tuple_type| {
12268                const size = wip_vi.size(isel);
12269                if (size <= 16 * 4) homogeneous_aggregate: {
12270                    const fdt = homogeneousTupleBaseType(zcu, tuple_type) orelse break :homogeneous_aggregate;
12271                    const parts_len = @shrExact(size, fdt.log2Size());
12272                    if (parts_len > 4) break :homogeneous_aggregate;
12273                    it.vectors(isel, wip_vi, fdt, @intCast(parts_len));
12274                    break :type_key;
12275                }
12276                switch (size) {
12277                    0 => unreachable,
12278                    1...8 => it.integer(isel, wip_vi),
12279                    9...16 => {
12280                        var part_offset: u64 = 0;
12281                        var part_sizes: [2]u64 = undefined;
12282                        var parts_len: Value.PartsLen = 0;
12283                        var next_field_end: u64 = 0;
12284                        var field_index: usize = 0;
12285                        while (part_offset < size) {
12286                            const field_end = next_field_end;
12287                            const next_field_begin = while (field_index < tuple_type.types.len) {
12288                                defer field_index += 1;
12289                                if (tuple_type.values.get(ip)[field_index] != .none) continue;
12290                                const field_ty: ZigType = .fromInterned(tuple_type.types.get(ip)[field_index]);
12291                                const next_field_begin = field_ty.abiAlignment(zcu).forward(field_end);
12292                                next_field_end = next_field_begin + field_ty.abiSize(zcu);
12293                                break next_field_begin;
12294                            } else std.mem.alignForward(u64, size, 8);
12295                            while (next_field_begin - part_offset >= 8) {
12296                                const part_size = @min(field_end - part_offset, 8);
12297                                part_sizes[parts_len] = part_size;
12298                                assert(part_offset + part_size <= size);
12299                                parts_len += 1;
12300                                part_offset += part_size;
12301                                if (part_offset >= field_end) part_offset = next_field_begin;
12302                            }
12303                        }
12304                        assert(parts_len == part_sizes.len);
12305                        it.integers(isel, wip_vi, part_sizes);
12306                    },
12307                    else => it.indirect(isel, wip_vi),
12308                }
12309            },
12310            .union_type => {
12311                const loaded_union = ip.loadUnionType(ty.toIntern());
12312                switch (loaded_union.flagsUnordered(ip).layout) {
12313                    .auto, .@"extern" => {},
12314                    .@"packed" => continue :type_key .{ .int_type = .{
12315                        .signedness = .unsigned,
12316                        .bits = @intCast(ty.bitSize(zcu)),
12317                    } },
12318                }
12319                switch (wip_vi.size(isel)) {
12320                    0 => unreachable,
12321                    1...8 => it.integer(isel, wip_vi),
12322                    9...16 => {
12323                        const union_layout = ZigType.getUnionLayout(loaded_union, zcu);
12324                        var sizes: [2]u64 = @splat(0);
12325                        {
12326                            const offset = union_layout.tagOffset();
12327                            const end = offset % 8 + union_layout.tag_size;
12328                            const part_index: usize = @intCast(offset / 8);
12329                            sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12330                            if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12331                        }
12332                        {
12333                            const offset = union_layout.payloadOffset();
12334                            const end = offset % 8 + union_layout.payload_size;
12335                            const part_index: usize = @intCast(offset / 8);
12336                            sizes[part_index] = @max(sizes[part_index], @min(end, 8));
12337                            if (end > 8) sizes[part_index + 1] = @max(sizes[part_index + 1], end - 8);
12338                        }
12339                        it.integers(isel, wip_vi, sizes);
12340                    },
12341                    else => it.indirect(isel, wip_vi),
12342                }
12343            },
12344            .opaque_type, .func_type => continue :type_key .{ .simple_type = .anyopaque },
12345            .enum_type => continue :type_key ip.indexToKey(ip.loadEnumType(ty.toIntern()).tag_ty),
12346            .error_set_type,
12347            .inferred_error_set_type,
12348            => continue :type_key .{ .simple_type = .anyerror },
12349            .undef,
12350            .simple_value,
12351            .variable,
12352            .@"extern",
12353            .func,
12354            .int,
12355            .err,
12356            .error_union,
12357            .enum_literal,
12358            .enum_tag,
12359            .empty_enum_value,
12360            .float,
12361            .ptr,
12362            .slice,
12363            .opt,
12364            .aggregate,
12365            .un,
12366            .memoized_call,
12367            => unreachable, // values, not types
12368        }
12369        return wip_vi.ref(isel);
12370    }
12371
12372    pub fn nonSysvVarArg(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12373        const ngrn = it.ngrn;
12374        defer it.ngrn = ngrn;
12375        it.ngrn = ngrn_end;
12376        const nsrn = it.nsrn;
12377        defer it.nsrn = nsrn;
12378        it.nsrn = nsrn_end;
12379        return it.param(isel, ty);
12380    }
12381
12382    pub fn ret(it: *CallAbiIterator, isel: *Select, ty: ZigType) !?Value.Index {
12383        const wip_vi = try it.param(isel, ty) orelse return null;
12384        switch (wip_vi.parent(isel)) {
12385            .unallocated, .stack_slot => {},
12386            .value, .constant => unreachable,
12387            .address => |address_vi| {
12388                assert(address_vi.hint(isel) == ngrn_start);
12389                address_vi.setHint(isel, ngrn_end);
12390            },
12391        }
12392        return wip_vi;
12393    }
12394
12395    pub const FundamentalDataType = enum {
12396        half,
12397        single,
12398        double,
12399        quad,
12400        vector64,
12401        vector128,
12402        fn log2Size(fdt: FundamentalDataType) u3 {
12403            return switch (fdt) {
12404                .half => 1,
12405                .single => 2,
12406                .double, .vector64 => 3,
12407                .quad, .vector128 => 4,
12408            };
12409        }
12410        fn size(fdt: FundamentalDataType) u64 {
12411            return @as(u64, 1) << fdt.log2Size();
12412        }
12413    };
12414    fn homogeneousAggregateBaseType(zcu: *Zcu, initial_ty: InternPool.Index) ?FundamentalDataType {
12415        const ip = &zcu.intern_pool;
12416        var ty = initial_ty;
12417        return type_key: switch (ip.indexToKey(ty)) {
12418            else => null,
12419            .array_type => |array_type| {
12420                ty = array_type.child;
12421                continue :type_key ip.indexToKey(ty);
12422            },
12423            .vector_type => switch (ZigType.fromInterned(ty).abiSize(zcu)) {
12424                else => null,
12425                8 => .vector64,
12426                16 => .vector128,
12427            },
12428            .simple_type => |simple_type| switch (simple_type) {
12429                .f16 => .half,
12430                .f32 => .single,
12431                .f64 => .double,
12432                .f128 => .quad,
12433                .c_longdouble => switch (zcu.getTarget().cTypeBitSize(.longdouble)) {
12434                    else => unreachable,
12435                    16 => .half,
12436                    32 => .single,
12437                    64 => .double,
12438                    80 => null,
12439                    128 => .quad,
12440                },
12441                else => null,
12442            },
12443            .struct_type => homogeneousStructBaseType(zcu, &ip.loadStructType(ty)),
12444            .tuple_type => |tuple_type| homogeneousTupleBaseType(zcu, tuple_type),
12445        };
12446    }
12447    fn homogeneousStructBaseType(zcu: *Zcu, loaded_struct: *const InternPool.LoadedStructType) ?FundamentalDataType {
12448        const ip = &zcu.intern_pool;
12449        var common_fdt: ?FundamentalDataType = null;
12450        for (0.., loaded_struct.field_types.get(ip)) |field_index, field_ty| {
12451            if (loaded_struct.fieldIsComptime(ip, field_index)) continue;
12452            if (loaded_struct.fieldAlign(ip, field_index) != .none) return null;
12453            if (!ZigType.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
12454            const fdt = homogeneousAggregateBaseType(zcu, field_ty);
12455            if (common_fdt == null) common_fdt = fdt else if (fdt != common_fdt) return null;
12456        }
12457        return common_fdt;
12458    }
12459    fn homogeneousTupleBaseType(zcu: *Zcu, tuple_type: InternPool.Key.TupleType) ?FundamentalDataType {
12460        const ip = &zcu.intern_pool;
12461        var common_fdt: ?FundamentalDataType = null;
12462        for (tuple_type.values.get(ip), tuple_type.types.get(ip)) |field_val, field_ty| {
12463            if (field_val != .none) continue;
12464            const fdt = homogeneousAggregateBaseType(zcu, field_ty);
12465            if (common_fdt == null) common_fdt = fdt else if (fdt != common_fdt) return null;
12466        }
12467        return common_fdt;
12468    }
12469
12470    const Spec = struct {
12471        offset: u64,
12472        size: u64,
12473    };
12474
12475    fn stack(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12476        // C.12
12477        it.nsaa = @intCast(wip_vi.alignment(isel).forward(it.nsaa));
12478        const parent_vi = switch (wip_vi.parent(isel)) {
12479            .unallocated, .stack_slot => wip_vi,
12480            .address, .constant => unreachable,
12481            .value => |parent_vi| parent_vi,
12482        };
12483        switch (parent_vi.parent(isel)) {
12484            .unallocated => parent_vi.setParent(isel, .{ .stack_slot = .{
12485                .base = .sp,
12486                .offset = it.nsaa,
12487            } }),
12488            .stack_slot => {},
12489            .address, .value, .constant => unreachable,
12490        }
12491        it.nsaa += @intCast(wip_vi.size(isel));
12492    }
12493
12494    fn integer(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12495        assert(wip_vi.size(isel) <= 8);
12496        const natural_alignment = wip_vi.alignment(isel);
12497        assert(natural_alignment.order(.@"16").compare(.lte));
12498        wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12499        if (it.ngrn == ngrn_end) return it.stack(isel, wip_vi);
12500        wip_vi.setHint(isel, it.ngrn);
12501        it.ngrn = @enumFromInt(@intFromEnum(it.ngrn) + 1);
12502    }
12503
12504    fn integers(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index, part_sizes: [2]u64) void {
12505        assert(wip_vi.size(isel) <= 16);
12506        const natural_alignment = wip_vi.alignment(isel);
12507        assert(natural_alignment.order(.@"16").compare(.lte));
12508        wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12509        // C.8
12510        if (natural_alignment == .@"16") it.ngrn = @enumFromInt(std.mem.alignForward(
12511            @typeInfo(Register.Alias).@"enum".tag_type,
12512            @intFromEnum(it.ngrn),
12513            2,
12514        ));
12515        if (it.ngrn == ngrn_end) return it.stack(isel, wip_vi);
12516        wip_vi.setParts(isel, part_sizes.len);
12517        for (0.., part_sizes) |part_index, part_size|
12518            it.integer(isel, wip_vi.addPart(isel, 8 * part_index, part_size));
12519    }
12520
12521    fn vector(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12522        assert(wip_vi.size(isel) <= 16);
12523        const natural_alignment = wip_vi.alignment(isel);
12524        assert(natural_alignment.order(.@"16").compare(.lte));
12525        wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12526        wip_vi.setIsVector(isel);
12527        if (it.nsrn == nsrn_end) return it.stack(isel, wip_vi);
12528        wip_vi.setHint(isel, it.nsrn);
12529        it.nsrn = @enumFromInt(@intFromEnum(it.nsrn) + 1);
12530    }
12531
12532    fn vectors(
12533        it: *CallAbiIterator,
12534        isel: *Select,
12535        wip_vi: Value.Index,
12536        fdt: FundamentalDataType,
12537        parts_len: Value.PartsLen,
12538    ) void {
12539        const fdt_log2_size = fdt.log2Size();
12540        assert(wip_vi.size(isel) == @shlExact(@as(u9, parts_len), fdt_log2_size));
12541        const natural_alignment = wip_vi.alignment(isel);
12542        assert(natural_alignment.order(.@"16").compare(.lte));
12543        wip_vi.setAlignment(isel, natural_alignment.maxStrict(.@"8"));
12544        if (@intFromEnum(it.nsrn) > @intFromEnum(nsrn_end) - parts_len) return it.stack(isel, wip_vi);
12545        if (parts_len == 1) return it.vector(isel, wip_vi);
12546        wip_vi.setParts(isel, parts_len);
12547        const fdt_size = @as(u64, 1) << fdt_log2_size;
12548        for (0..parts_len) |part_index|
12549            it.vector(isel, wip_vi.addPart(isel, part_index << fdt_log2_size, fdt_size));
12550    }
12551
12552    fn indirect(it: *CallAbiIterator, isel: *Select, wip_vi: Value.Index) void {
12553        const wip_address_vi = isel.initValue(.usize);
12554        wip_vi.setParent(isel, .{ .address = wip_address_vi });
12555        it.integer(isel, wip_address_vi);
12556    }
12557};
12558
12559const Air = @import("../../Air.zig");
12560const assert = std.debug.assert;
12561const codegen = @import("../../codegen.zig");
12562const Constant = @import("../../Value.zig");
12563const InternPool = @import("../../InternPool.zig");
12564const Package = @import("../../Package.zig");
12565const Register = codegen.aarch64.encoding.Register;
12566const Select = @This();
12567const std = @import("std");
12568const tracking_log = std.log.scoped(.tracking);
12569const wip_mir_log = std.log.scoped(.@"wip-mir");
12570const Zcu = @import("../../Zcu.zig");
12571const ZigType = @import("../../Type.zig");