Commit 0d9079f466
Changed files (4)
src
arch
x86_64
test
behavior
src/arch/x86_64/CodeGen.zig
@@ -2492,8 +2492,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
.atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst),
.array_elem_val => try cg.airArrayElemVal(inst),
- .slice_elem_val => try cg.airSliceElemVal(inst),
- .ptr_elem_val => try cg.airPtrElemVal(inst),
.optional_payload => try cg.airOptionalPayload(inst),
.unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst),
@@ -3995,7 +3993,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .load_frame = .{
.index = .ret_addr,
} });
- while (try slot.toAnyReg(cg)) {}
+ while (try slot.toRegClass(true, .general_purpose, cg)) {}
try slot.moveTo(inst, cg);
},
.frame_addr => if (use_old) try cg.airFrameAddress(inst) else {
@@ -9445,7 +9443,111 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
try ops[0].toOffset(0, cg);
try ops[0].moveTo(inst, cg);
},
- .slice_elem_ptr, .ptr_elem_ptr => |tag| if (use_old) switch (tag) {
+ .slice_elem_val, .ptr_elem_val => |air_tag| if (use_old) switch (air_tag) {
+ else => unreachable,
+ .slice_elem_val => try cg.airSliceElemVal(inst),
+ .ptr_elem_val => try cg.airPtrElemVal(inst),
+ } else {
+ const bin_op = air_datas[@intFromEnum(inst)].bin_op;
+ var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
+ switch (air_tag) {
+ else => unreachable,
+ .slice_elem_val => try ops[0].toLimb(0, cg),
+ .ptr_elem_val => {},
+ }
+ var res: [1]Temp = undefined;
+ const res_ty = cg.typeOfIndex(inst);
+ cg.select(&res, &.{res_ty}, &ops, comptime &.{ .{
+ .dst_constraints = .{.{ .int = .byte }},
+ .patterns = &.{
+ .{ .src = .{ .to_gpr, .simm32 } },
+ .{ .src = .{ .to_gpr, .to_gpr } },
+ },
+ .dst_temps = .{.{ .rc = .general_purpose }},
+ .each = .{ .once = &.{
+ .{ ._, ._, .movzx, .dst0d, .leai(.byte, .src0, .src1), ._, ._ },
+ } },
+ }, .{
+ .dst_constraints = .{.{ .int = .word }},
+ .patterns = &.{
+ .{ .src = .{ .to_gpr, .simm32 } },
+ .{ .src = .{ .to_gpr, .to_gpr } },
+ },
+ .dst_temps = .{.{ .rc = .general_purpose }},
+ .each = .{ .once = &.{
+ .{ ._, ._, .movzx, .dst0d, .leasi(.word, .src0, .@"2", .src1), ._, ._ },
+ } },
+ }, .{
+ .dst_constraints = .{.{ .int = .dword }},
+ .patterns = &.{
+ .{ .src = .{ .to_gpr, .simm32 } },
+ .{ .src = .{ .to_gpr, .to_gpr } },
+ },
+ .dst_temps = .{.{ .rc = .general_purpose }},
+ .each = .{ .once = &.{
+ .{ ._, ._, .mov, .dst0d, .leasi(.dword, .src0, .@"4", .src1), ._, ._ },
+ } },
+ }, .{
+ .required_features = .{ .@"64bit", null, null, null },
+ .dst_constraints = .{.{ .int = .qword }},
+ .patterns = &.{
+ .{ .src = .{ .to_gpr, .simm32 } },
+ .{ .src = .{ .to_gpr, .to_gpr } },
+ },
+ .dst_temps = .{.{ .rc = .general_purpose }},
+ .each = .{ .once = &.{
+ .{ ._, ._, .mov, .dst0q, .leasi(.qword, .src0, .@"8", .src1), ._, ._ },
+ } },
+ } }) catch |err| switch (err) {
+ error.SelectFailed => switch (res_ty.abiSize(zcu)) {
+ 0 => res[0] = try cg.tempFromValue(res_ty, .none),
+ else => |elem_size| {
+ while (true) for (&ops) |*op| {
+ if (try op.toRegClass(true, .general_purpose, cg)) break;
+ } else break;
+ const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
+ const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
+ if (!std.math.isPowerOfTwo(elem_size)) {
+ try cg.spillEflagsIfOccupied();
+ try cg.asmRegisterRegisterImmediate(
+ .{ .i_, .mul },
+ rhs_reg,
+ rhs_reg,
+ .u(elem_size),
+ );
+ try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
+ .base = .{ .reg = lhs_reg },
+ .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
+ });
+ } else if (elem_size > 8) {
+ try cg.spillEflagsIfOccupied();
+ try cg.asmRegisterImmediate(
+ .{ ._l, .sh },
+ rhs_reg,
+ .u(std.math.log2_int(u64, elem_size)),
+ );
+ try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
+ .base = .{ .reg = lhs_reg },
+ .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } },
+ });
+ } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{
+ .base = .{ .reg = lhs_reg },
+ .mod = .{ .rm = .{
+ .size = .qword,
+ .index = rhs_reg,
+ .scale = .fromFactor(@intCast(elem_size)),
+ } },
+ });
+ res[0] = try ops[0].load(res_ty, cg);
+ },
+ },
+ else => |e| return e,
+ };
+ if (ops[0].index != res[0].index) try ops[0].die(cg);
+ if (ops[1].index != res[0].index) try ops[1].die(cg);
+ try res[0].moveTo(inst, cg);
+ },
+ .slice_elem_ptr, .ptr_elem_ptr => |air_tag| if (use_old) switch (air_tag) {
else => unreachable,
.slice_elem_ptr => try cg.airSliceElemPtr(inst),
.ptr_elem_ptr => try cg.airPtrElemPtr(inst),
@@ -9453,7 +9555,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
- switch (tag) {
+ switch (air_tag) {
else => unreachable,
.slice_elem_ptr => try ops[0].toLimb(0, cg),
.ptr_elem_ptr => {},
@@ -9463,7 +9565,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
const elem_size = dst_ty.childType(zcu).abiSize(zcu);
if (elem_size == 0) break :zero_offset;
while (true) for (&ops) |*op| {
- if (try op.toAnyReg(cg)) break;
+ if (try op.toRegClass(true, .general_purpose, cg)) break;
} else break;
const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64();
const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64();
@@ -27718,6 +27820,7 @@ const Temp = struct {
},
};
const new_temp_index = cg.next_temp_index;
+ try cg.register_manager.getReg(new_reg, new_temp_index.toIndex());
cg.temp_type[@intFromEnum(new_temp_index)] = ty;
try cg.genSetReg(new_reg, ty, val, .{});
new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
@@ -27727,27 +27830,6 @@ const Temp = struct {
return true;
}
- fn toAnyReg(temp: *Temp, cg: *CodeGen) !bool {
- const val, const ty = switch (temp.unwrap(cg)) {
- .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
- .temp => |temp_index| val: {
- const temp_tracking = temp_index.tracking(cg);
- if (temp_tracking.short == .register) return false;
- break :val .{ temp_tracking.short, temp_index.typeOf(cg) };
- },
- };
- const new_temp_index = cg.next_temp_index;
- cg.temp_type[@intFromEnum(new_temp_index)] = ty;
- const new_reg =
- try cg.register_manager.allocReg(new_temp_index.toIndex(), cg.regSetForType(ty));
- try cg.genSetReg(new_reg, ty, val, .{});
- new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
- try temp.die(cg);
- cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
- temp.* = .{ .index = new_temp_index.toIndex() };
- return true;
- }
-
fn toRegClass(temp: *Temp, mut: bool, rc: Register.Class, cg: *CodeGen) !bool {
const val = temp.tracking(cg).short;
if (!mut or temp.isMut(cg)) switch (val) {
@@ -27769,7 +27851,7 @@ const Temp = struct {
fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
- if (try part_temp.toAnyReg(cg)) break;
+ if (try part_temp.toRegClass(true, .general_purpose, cg)) break;
} else break;
const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
@@ -27824,12 +27906,12 @@ const Temp = struct {
.load_got,
.load_tlv,
.load_frame,
- => return temp.toAnyReg(cg),
+ => return temp.toRegClass(true, .general_purpose, cg),
.lea_symbol => |sym_off| {
const off = sym_off.off;
if (off == 0) return false;
try temp.toOffset(-off, cg);
- while (try temp.toAnyReg(cg)) {}
+ while (try temp.toRegClass(true, .general_purpose, cg)) {}
try temp.toOffset(off, cg);
return true;
},
@@ -27868,24 +27950,16 @@ const Temp = struct {
}
fn load(ptr: *Temp, val_ty: Type, cg: *CodeGen) !Temp {
- const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu));
const val = try cg.tempAlloc(val_ty);
switch (val.tracking(cg).short) {
else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
.register => |val_reg| {
while (try ptr.toLea(cg)) {}
- switch (val_reg.class()) {
- .general_purpose => try cg.asmRegisterMemory(
- .{ ._, .mov },
- registerAlias(val_reg, val_abi_size),
- try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
- ),
- else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
- }
+ try cg.genSetReg(val_reg, val_ty, ptr.tracking(cg).short.deref(), .{});
},
.load_frame => |val_frame_addr| {
var val_ptr = try cg.tempFromValue(.usize, .{ .lea_frame = val_frame_addr });
- var len = try cg.tempFromValue(.usize, .{ .immediate = val_abi_size });
+ var len = try cg.tempFromValue(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });
try val_ptr.memcpy(ptr, &len, cg);
try val_ptr.die(cg);
try len.die(cg);
@@ -27908,7 +27982,7 @@ const Temp = struct {
);
} else continue :val .{ .register = undefined },
.register => {
- while (try ptr.toLea(cg) or try val.toAnyReg(cg)) {}
+ while (try ptr.toLea(cg) or try val.toRegClass(true, .general_purpose, cg)) {}
const val_reg = val.tracking(cg).short.register;
switch (val_reg.class()) {
.general_purpose => try cg.asmMemoryRegister(
@@ -28224,6 +28298,7 @@ const Select = struct {
any_int,
any_signed_int,
any_float,
+ po2_any,
bool_vec: Memory.Size,
vec: Memory.Size,
signed_int_vec: Memory.Size,
@@ -28250,15 +28325,17 @@ const Select = struct {
unsigned_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size },
signed_int: Memory.Size,
unsigned_int: Memory.Size,
+ elem_int: Memory.Size,
fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool {
const zcu = cg.pt.zcu;
switch (constraint) {
.any => return true,
- .any_bool_vec => return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type,
+ .any_bool_vec => return ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type,
.any_int => return ty.toIntern() == .bool_type or ty.isPtrAtRuntime(zcu) or ty.isAbiInt(zcu),
.any_signed_int => return ty.isAbiInt(zcu) and ty.intInfo(zcu).signedness == .signed,
- .any_float => return ty.scalarType(zcu).isRuntimeFloat(),
+ .any_float => return ty.isRuntimeFloat(),
+ .po2_any => return std.math.isPowerOfTwo(ty.abiSize(zcu)),
.bool_vec => |size| return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and
size.bitSize(cg.target) >= ty.vectorLen(zcu),
.vec => |size| return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() != .bool_type and
@@ -28434,6 +28511,12 @@ const Select = struct {
const int_info = ty.intInfo(zcu);
return int_info.signedness == .unsigned and size.bitSize(cg.target) >= int_info.bits;
},
+ .elem_int => |size| {
+ const elem_ty = ty.childType(zcu);
+ if (elem_ty.toIntern() == .bool_type) return true;
+ if (elem_ty.isPtrAtRuntime(zcu)) return size.bitSize(cg.target) >= cg.target.ptrBitWidth();
+ return elem_ty.isAbiInt(zcu) and size.bitSize(cg.target) >= elem_ty.intInfo(zcu).bits;
+ },
}
}
};
@@ -29107,7 +29190,14 @@ const Select = struct {
const UnsignedImm = @Type(.{
.int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits },
});
- return op.imm + @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) {
+ return switch (op.index.ref) {
+ else => |ref| switch (ref.deref(s).tracking(s.cg).short) {
+ else => unreachable,
+ .immediate => |imm| op.index.scale.toFactor() * @as(i32, @intCast(imm)),
+ .register => 0,
+ },
+ .none => 0,
+ } + @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) {
.none => 0,
.ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8),
.ptr_bit_size => s.cg.target.ptrBitWidth(),
@@ -29120,7 +29210,7 @@ const Select = struct {
op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu),
@divExact(op.base.size.bitSize(s.cg.target), 8),
)),
- .src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
+ .src0_elem_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).childType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)),
.smin => @as(SignedImm, std.math.minInt(SignedImm)) >> @truncate(
-%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
),
@@ -29130,7 +29220,7 @@ const Select = struct {
.umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate(
-%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu),
)),
- });
+ }) + op.imm;
}
fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand {
@@ -29160,7 +29250,11 @@ const Select = struct {
.mod = .{ .rm = .{
.size = op.base.size,
.index = switch (op.index.ref) {
- else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
+ else => |ref| switch (ref.deref(s).tracking(s.cg).short) {
+ else => unreachable,
+ .immediate => .none,
+ .register => |index_reg| registerAlias(index_reg, @divExact(s.cg.target.ptrBitWidth(), 8)),
+ },
.none => .none,
},
.scale = op.index.scale,
@@ -29170,7 +29264,11 @@ const Select = struct {
.mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{
.size = op.base.size,
.index = switch (op.index.ref) {
- else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)),
+ else => |ref| switch (ref.deref(s).tracking(s.cg).short) {
+ else => unreachable,
+ .immediate => .none,
+ .register => |index_reg| registerAlias(index_reg, @divExact(s.cg.target.ptrBitWidth(), 8)),
+ },
.none => .none,
},
.scale = op.index.scale,
test/behavior/x86_64/build.zig
@@ -88,6 +88,7 @@ pub fn build(b: *std.Build) void {
const cpu = query.serializeCpuAlloc(b.allocator) catch @panic("OOM");
for ([_][]const u8{
"math.zig",
+ "mem.zig",
}) |path| {
const test_mod = b.createModule(.{
.root_source_file = b.path(path),
test/behavior/x86_64/mem.zig
@@ -0,0 +1,31 @@
+fn access(comptime array: anytype) !void {
+ var slice: []const @typeInfo(@TypeOf(array)).array.child = undefined;
+ slice = &array;
+ inline for (0.., &array) |ct_index, *elem| {
+ var rt_index: usize = undefined;
+ rt_index = ct_index;
+ if (&slice.ptr[ct_index] != elem) return error.Unexpected;
+ if (&slice[ct_index] != elem) return error.Unexpected;
+ if (&slice.ptr[rt_index] != elem) return error.Unexpected;
+ if (&slice[rt_index] != elem) return error.Unexpected;
+ if (slice.ptr[ct_index] != elem.*) return error.Unexpected;
+ if (slice[ct_index] != elem.*) return error.Unexpected;
+ if (slice.ptr[rt_index] != elem.*) return error.Unexpected;
+ if (slice[rt_index] != elem.*) return error.Unexpected;
+ }
+}
+test access {
+ try access([3]u8{ 0xdb, 0xef, 0xbd });
+ try access([3]u16{ 0x340e, 0x3654, 0x88d7 });
+ try access([3]u32{ 0xd424c2c0, 0x2d6ac466, 0x5a0cfaba });
+ try access([3]u64{
+ 0x9327a4f5221666a6,
+ 0x5c34d3ddd84a8b12,
+ 0xbae087f39f649260,
+ });
+ try access([3]u128{
+ 0x601cf010065444d4d42d5536dd9b95db,
+ 0xa03f592fcaa22d40af23a0c735531e3c,
+ 0x5da44907b31602b95c2d93f0b582ceab,
+ });
+}
test/behavior/x86_64.zig
@@ -5,4 +5,5 @@ test {
if (builtin.zig_backend != .stage2_x86_64) return error.SkipZigTest;
if (builtin.object_format == .coff) return error.SkipZigTest;
_ = @import("x86_64/math.zig");
+ _ = @import("x86_64/mem.zig");
}