Commit 6e72026e3b
Changed files (2)
src
Air
codegen
src/Air/Legalize.zig
@@ -1,7 +1,36 @@
pt: Zcu.PerThread,
air_instructions: std.MultiArrayList(Air.Inst),
air_extra: std.ArrayListUnmanaged(u32),
-features: *const Features,
+features: if (switch (dev.env) {
+ .bootstrap => @import("../codegen/c.zig").legalizeFeatures(undefined),
+ else => null,
+}) |bootstrap_features| struct {
+ fn init(features: *const Features) @This() {
+ assert(features.eql(bootstrap_features.*));
+ return .{};
+ }
+ /// `inline` to propagate comptime-known result.
+ inline fn has(_: @This(), comptime feature: Feature) bool {
+ return comptime bootstrap_features.contains(feature);
+ }
+ /// `inline` to propagate comptime-known result.
+ fn hasAny(_: @This(), comptime features: []const Feature) bool {
+ return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());
+ }
+} else struct {
+ features: *const Features,
+ /// `inline` to propagate whether `dev.check` returns.
+ inline fn init(features: *const Features) @This() {
+ dev.check(.legalize);
+ return .{ .features = features };
+ }
+ fn has(rt: @This(), comptime feature: Feature) bool {
+ return rt.features.contains(feature);
+ }
+ fn hasAny(rt: @This(), comptime features: []const Feature) bool {
+ return !rt.features.intersectWith(comptime .initMany(features)).eql(comptime .initEmpty());
+ }
+},
pub const Feature = enum {
scalarize_add,
@@ -199,7 +228,7 @@ pub const Feature = enum {
.float_from_int => .scalarize_float_from_int,
.shuffle_one => .scalarize_shuffle_one,
.shuffle_two => .scalarize_shuffle_two,
- .select => .scalarize_selects,
+ .select => .scalarize_select,
.mul_add => .scalarize_mul_add,
};
}
@@ -210,13 +239,12 @@ pub const Features = std.enums.EnumSet(Feature);
pub const Error = std.mem.Allocator.Error;
pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void {
- dev.check(.legalize);
assert(!features.eql(comptime .initEmpty())); // backend asked to run legalize, but no features were enabled
var l: Legalize = .{
.pt = pt,
.air_instructions = air.instructions.toMultiArrayList(),
.air_extra = air.extra,
- .features = features,
+ .features = .init(features),
};
defer air.* = l.getTmpAir();
const main_extra = l.extraData(Air.Block, l.air_extra.items[@intFromEnum(Air.ExtraIndex.main_block)]);
@@ -278,28 +306,28 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.bit_and,
.bit_or,
.xor,
- => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) {
+ => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
},
- .add_safe => if (l.features.contains(.expand_add_safe)) {
- assert(!l.features.contains(.scalarize_add_safe)); // it doesn't make sense to do both
+ .add_safe => if (l.features.has(.expand_add_safe)) {
+ assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
- } else if (l.features.contains(.scalarize_add_safe)) {
+ } else if (l.features.has(.scalarize_add_safe)) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
},
- .sub_safe => if (l.features.contains(.expand_sub_safe)) {
- assert(!l.features.contains(.scalarize_sub_safe)); // it doesn't make sense to do both
+ .sub_safe => if (l.features.has(.expand_sub_safe)) {
+ assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
- } else if (l.features.contains(.scalarize_sub_safe)) {
+ } else if (l.features.has(.scalarize_sub_safe)) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
},
- .mul_safe => if (l.features.contains(.expand_mul_safe)) {
- assert(!l.features.contains(.scalarize_mul_safe)); // it doesn't make sense to do both
+ .mul_safe => if (l.features.has(.expand_mul_safe)) {
+ assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
- } else if (l.features.contains(.scalarize_mul_safe)) {
+ } else if (l.features.has(.scalarize_mul_safe)) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
},
@@ -308,7 +336,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.sub_with_overflow,
.mul_with_overflow,
.shl_with_overflow,
- => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) {
+ => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
},
@@ -320,13 +348,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.shl,
.shl_exact,
.shl_sat,
- => |air_tag| if (!l.features.intersectWith(comptime .initMany(&.{
+ => |air_tag| if (l.features.hasAny(&.{
.unsplat_shift_rhs,
.scalarize(air_tag),
- })).eql(comptime .initEmpty())) {
+ })) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.rhs).isVector(zcu)) {
- if (l.features.contains(.unsplat_shift_rhs)) {
+ if (l.features.has(.unsplat_shift_rhs)) {
if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) {
else => {},
.aggregate => |aggregate| switch (aggregate.storage) {
@@ -347,7 +375,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
}
}
}
- if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);
+ if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);
}
},
inline .not,
@@ -364,11 +392,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.int_from_float,
.int_from_float_optimized,
.float_from_int,
- => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) {
+ => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
},
- .bitcast => if (l.features.contains(.scalarize_bitcast)) {
+ .bitcast => if (l.features.has(.scalarize_bitcast)) {
const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
const to_ty = ty_op.ty.toType();
@@ -404,10 +432,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
};
if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst));
},
- .intcast_safe => if (l.features.contains(.expand_intcast_safe)) {
- assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both
+ .intcast_safe => if (l.features.has(.expand_intcast_safe)) {
+ assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
- } else if (l.features.contains(.scalarize_intcast_safe)) {
+ } else if (l.features.has(.scalarize_intcast_safe)) {
const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
},
@@ -442,7 +470,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.trunc_float,
.neg,
.neg_optimized,
- => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) {
+ => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op);
},
@@ -459,7 +487,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.cmp_neq,
.cmp_neq_optimized,
=> {},
- inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) {
+ inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector);
},
@@ -513,13 +541,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.bool_and,
.bool_or,
=> {},
- .load => if (l.features.contains(.expand_packed_load)) {
+ .load => if (l.features.has(.expand_packed_load)) {
const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
},
.ret, .ret_safe, .ret_load => {},
- .store, .store_safe => if (l.features.contains(.expand_packed_store)) {
+ .store, .store_safe => if (l.features.has(.expand_packed_store)) {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
@@ -542,7 +570,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.struct_field_ptr_index_2,
.struct_field_ptr_index_3,
=> {},
- .struct_field_val => if (l.features.contains(.expand_packed_struct_field_val)) {
+ .struct_field_val => if (l.features.has(.expand_packed_struct_field_val)) {
const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
const extra = l.extraData(Air.StructField, ty_pl.payload).data;
switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) {
@@ -564,7 +592,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.ptr_elem_ptr,
.array_to_slice,
=> {},
- .reduce, .reduce_optimized => if (l.features.contains(.reduce_one_elem_to_bitcast)) {
+ .reduce, .reduce_optimized => if (l.features.has(.reduce_one_elem_to_bitcast)) {
const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
const vector_ty = l.typeOf(reduce.operand);
switch (vector_ty.vectorLen(zcu)) {
@@ -577,9 +605,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
}
},
.splat => {},
- .shuffle_one => if (l.features.contains(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),
- .shuffle_two => if (l.features.contains(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),
- .select => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select),
+ .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),
+ .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),
+ .select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select),
.memset,
.memset_safe,
.memcpy,
@@ -597,7 +625,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.error_name,
.error_set_has_value,
=> {},
- .aggregate_init => if (l.features.contains(.expand_packed_aggregate_init)) {
+ .aggregate_init => if (l.features.has(.expand_packed_aggregate_init)) {
const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
const agg_ty = ty_pl.ty.toType();
switch (agg_ty.zigTypeTag(zcu)) {
@@ -609,7 +637,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
}
},
.union_init, .prefetch => {},
- .mul_add => if (l.features.contains(.scalarize_mul_add)) {
+ .mul_add => if (l.features.has(.scalarize_mul_add)) {
const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin);
},
@@ -636,6 +664,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
}
const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select };
+/// inline to propagate comptime-known `replaceInst` result.
inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
}
@@ -2691,7 +2720,7 @@ fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 {
}
/// Returns `tag` to remind the caller to `continue :inst` the result.
-/// This is inline to propagate the comptime-known `tag`.
+/// `inline` to propagate the comptime-known `tag` result.
inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {};
l.air_instructions.set(@intFromEnum(inst), .{ .tag = tag, .data = data });
src/codegen/c.zig
@@ -23,12 +23,19 @@ const BigIntLimb = std.math.big.Limb;
const BigInt = std.math.big.int;
pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
- return if (dev.env.supports(.legalize)) comptime &.initMany(&.{
- .expand_intcast_safe,
- .expand_add_safe,
- .expand_sub_safe,
- .expand_mul_safe,
- }) else null; // we don't currently ask zig1 to use safe optimization modes
+ return comptime switch (dev.env.supports(.legalize)) {
+ inline false, true => |supports_legalize| &.init(.{
+ // we don't currently ask zig1 to use safe optimization modes
+ .expand_intcast_safe = supports_legalize,
+ .expand_add_safe = supports_legalize,
+ .expand_sub_safe = supports_legalize,
+ .expand_mul_safe = supports_legalize,
+
+ .expand_packed_load = true,
+ .expand_packed_store = true,
+ .expand_packed_struct_field_val = true,
+ }),
+ };
}
/// For most backends, MIR is basically a sequence of machine code instructions, perhaps with some