Commit 4c4dacf81a
Changed files (10)
src
Air
arch
codegen
src/Air/Legalize.zig
@@ -81,6 +81,19 @@ pub const Feature = enum {
/// Legalize reduce of a one element vector to a bitcast
reduce_one_elem_to_bitcast,
+ /// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.
+ /// Not compatible with `scalarize_intcast_safe`.
+ expand_intcast_safe,
+ /// Replace `add_safe` with an explicit safety check which `call`s the panic function on failure.
+ /// Not compatible with `scalarize_add_safe`.
+ expand_add_safe,
+ /// Replace `sub_safe` with an explicit safety check which `call`s the panic function on failure.
+ /// Not compatible with `scalarize_sub_safe`.
+ expand_sub_safe,
+ /// Replace `mul_safe` with an explicit safety check which `call`s the panic function on failure.
+ /// Not compatible with `scalarize_mul_safe`.
+ expand_mul_safe,
+
fn scalarize(tag: Air.Inst.Tag) Feature {
return switch (tag) {
else => unreachable,
@@ -205,17 +218,14 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.arg,
=> {},
inline .add,
- .add_safe,
.add_optimized,
.add_wrap,
.add_sat,
.sub,
- .sub_safe,
.sub_optimized,
.sub_wrap,
.sub_sat,
.mul,
- .mul_safe,
.mul_optimized,
.mul_wrap,
.mul_sat,
@@ -240,6 +250,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
},
+ .add_safe => if (l.features.contains(.expand_add_safe)) {
+ assert(!l.features.contains(.scalarize_add_safe)); // it doesn't make sense to do both
+ continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
+ } else if (l.features.contains(.scalarize_add_safe)) {
+ const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
+ if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
+ },
+ .sub_safe => if (l.features.contains(.expand_sub_safe)) {
+ assert(!l.features.contains(.scalarize_sub_safe)); // it doesn't make sense to do both
+ continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
+ } else if (l.features.contains(.scalarize_sub_safe)) {
+ const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
+ if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
+ },
+ .mul_safe => if (l.features.contains(.expand_mul_safe)) {
+ assert(!l.features.contains(.scalarize_mul_safe)); // it doesn't make sense to do both
+ continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
+ } else if (l.features.contains(.scalarize_mul_safe)) {
+ const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
+ if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
+ },
.ptr_add,
.ptr_sub,
.add_with_overflow,
@@ -295,7 +326,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
.fptrunc,
.fpext,
.intcast,
- .intcast_safe,
.trunc,
.int_from_float,
.int_from_float_optimized,
@@ -312,6 +342,13 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
if (to_ty.isVector(zcu) and from_ty.isVector(zcu) and to_ty.vectorLen(zcu) == from_ty.vectorLen(zcu))
continue :inst try l.scalarize(inst, .ty_op);
},
+ .intcast_safe => if (l.features.contains(.expand_intcast_safe)) {
+ assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both
+ continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
+ } else if (l.features.contains(.scalarize_intcast_safe)) {
+ const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
+ if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
+ },
.block,
.loop,
=> {
@@ -550,81 +587,83 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
const expected_instructions_len = l.air_instructions.len + (6 + arity + 8);
try l.air_instructions.ensureTotalCapacity(gpa, expected_instructions_len);
- var res_block: Block(4) = .empty;
+ var res_block_buf: [4]Air.Inst.Index = undefined;
+ var res_block: Block = .init(&res_block_buf);
{
- const res_alloc_inst = res_block.add(l.addInstAssumeCapacity(.{
+ const res_alloc_inst = res_block.add(l, .{
.tag = .alloc,
.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
- }));
- const index_alloc_inst = res_block.add(l.addInstAssumeCapacity(.{
+ });
+ const index_alloc_inst = res_block.add(l, .{
.tag = .alloc,
.data = .{ .ty = .ptr_usize },
- }));
- _ = res_block.add(l.addInstAssumeCapacity(.{
+ });
+ _ = res_block.add(l, .{
.tag = .store,
.data = .{ .bin_op = .{
.lhs = index_alloc_inst.toRef(),
.rhs = .zero_usize,
} },
- }));
+ });
const loop_inst: Air.Inst.Index = @enumFromInt(l.air_instructions.len + (3 + arity + 7));
- var loop_block: Block(3 + arity + 2) = .empty;
+ var loop_block_buf: [3 + arity + 2]Air.Inst.Index = undefined;
+ var loop_block: Block = .init(&loop_block_buf);
{
- const cur_index_inst = loop_block.add(l.addInstAssumeCapacity(.{
+ const cur_index_inst = loop_block.add(l, .{
.tag = .load,
.data = .{ .ty_op = .{
.ty = .usize_type,
.operand = index_alloc_inst.toRef(),
} },
- }));
- _ = loop_block.add(l.addInstAssumeCapacity(.{
+ });
+ _ = loop_block.add(l, .{
.tag = .vector_store_elem,
.data = .{ .vector_store_elem = .{
.vector_ptr = res_alloc_inst.toRef(),
.payload = try l.addExtra(Air.Bin, .{
.lhs = cur_index_inst.toRef(),
- .rhs = loop_block.add(l.addInstAssumeCapacity(res_elem: switch (data_tag) {
+ .rhs = loop_block.add(l, res_elem: switch (data_tag) {
.un_op => .{
.tag = orig.tag,
- .data = .{ .un_op = loop_block.add(l.addInstAssumeCapacity(.{
+ .data = .{ .un_op = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.un_op,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef() },
+ }).toRef() },
},
.ty_op => .{
.tag = orig.tag,
.data = .{ .ty_op = .{
.ty = Air.internedToRef(orig.data.ty_op.ty.toType().scalarType(zcu).toIntern()),
- .operand = loop_block.add(l.addInstAssumeCapacity(.{
+ .operand = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.ty_op.operand,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
} },
},
.bin_op => .{
.tag = orig.tag,
.data = .{ .bin_op = .{
- .lhs = loop_block.add(l.addInstAssumeCapacity(.{
+ .lhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.bin_op.lhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
- .rhs = loop_block.add(l.addInstAssumeCapacity(.{
+ }).toRef(),
+ .rhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.bin_op.rhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
} },
},
.ty_pl_vector_cmp => {
@@ -650,20 +689,20 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
},
},
.data = .{ .bin_op = .{
- .lhs = loop_block.add(l.addInstAssumeCapacity(.{
+ .lhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.lhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
- .rhs = loop_block.add(l.addInstAssumeCapacity(.{
+ }).toRef(),
+ .rhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.rhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
} },
};
},
@@ -673,94 +712,96 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
.tag = orig.tag,
.data = .{ .pl_op = .{
.payload = try l.addExtra(Air.Bin, .{
- .lhs = loop_block.add(l.addInstAssumeCapacity(.{
+ .lhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.lhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
- .rhs = loop_block.add(l.addInstAssumeCapacity(.{
+ }).toRef(),
+ .rhs = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = extra.rhs,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
}),
- .operand = loop_block.add(l.addInstAssumeCapacity(.{
+ .operand = loop_block.add(l, .{
.tag = .array_elem_val,
.data = .{ .bin_op = .{
.lhs = orig.data.pl_op.operand,
.rhs = cur_index_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
} },
};
},
- })).toRef(),
+ }).toRef(),
}),
} },
- }));
- const not_done_inst = loop_block.add(l.addInstAssumeCapacity(.{
+ });
+ const not_done_inst = loop_block.add(l, .{
.tag = .cmp_lt,
.data = .{ .bin_op = .{
.lhs = cur_index_inst.toRef(),
.rhs = try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),
} },
- }));
+ });
- var not_done_block: Block(3) = .empty;
+ var not_done_block_buf: [3]Air.Inst.Index = undefined;
+ var not_done_block: Block = .init(¬_done_block_buf);
{
- _ = not_done_block.add(l.addInstAssumeCapacity(.{
+ _ = not_done_block.add(l, .{
.tag = .store,
.data = .{ .bin_op = .{
.lhs = index_alloc_inst.toRef(),
- .rhs = not_done_block.add(l.addInstAssumeCapacity(.{
+ .rhs = not_done_block.add(l, .{
.tag = .add,
.data = .{ .bin_op = .{
.lhs = cur_index_inst.toRef(),
.rhs = .one_usize,
} },
- })).toRef(),
+ }).toRef(),
} },
- }));
- _ = not_done_block.add(l.addInstAssumeCapacity(.{
+ });
+ _ = not_done_block.add(l, .{
.tag = .repeat,
.data = .{ .repeat = .{ .loop_inst = loop_inst } },
- }));
+ });
}
- var done_block: Block(2) = .empty;
+ var done_block_buf: [2]Air.Inst.Index = undefined;
+ var done_block: Block = .init(&done_block_buf);
{
- _ = done_block.add(l.addInstAssumeCapacity(.{
+ _ = done_block.add(l, .{
.tag = .br,
.data = .{ .br = .{
.block_inst = orig_inst,
- .operand = done_block.add(l.addInstAssumeCapacity(.{
+ .operand = done_block.add(l, .{
.tag = .load,
.data = .{ .ty_op = .{
.ty = Air.internedToRef(res_ty.toIntern()),
.operand = res_alloc_inst.toRef(),
} },
- })).toRef(),
+ }).toRef(),
} },
- }));
+ });
}
- _ = loop_block.add(l.addInstAssumeCapacity(.{
+ _ = loop_block.add(l, .{
.tag = .cond_br,
.data = .{ .pl_op = .{
.operand = not_done_inst.toRef(),
.payload = try l.addCondBrBodies(not_done_block.body(), done_block.body()),
} },
- }));
+ });
}
- assert(loop_inst == res_block.add(l.addInstAssumeCapacity(.{
+ assert(loop_inst == res_block.add(l, .{
.tag = .loop,
.data = .{ .ty_pl = .{
.ty = .noreturn_type,
.payload = try l.addBlockBody(loop_block.body()),
} },
- })));
+ }));
}
assert(l.air_instructions.len == expected_instructions_len);
return .{ .ty_pl = .{
@@ -768,29 +809,423 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime data_
.payload = try l.addBlockBody(res_block.body()),
} };
}
+fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
+ const pt = l.pt;
+ const zcu = pt.zcu;
+ const gpa = zcu.gpa;
+ const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
+
+ const operand_ref = ty_op.operand;
+ const operand_ty = l.typeOf(operand_ref);
+ const dest_ty = ty_op.ty.toType();
+
+ const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
+ const operand_scalar_ty = operand_ty.scalarType(zcu);
+ const dest_scalar_ty = dest_ty.scalarType(zcu);
+
+ assert(operand_scalar_ty.zigTypeTag(zcu) == .int);
+ const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) {
+ .int => false,
+ .@"enum" => true,
+ else => unreachable,
+ };
+
+ const operand_info = operand_scalar_ty.intInfo(zcu);
+ const dest_info = dest_scalar_ty.intInfo(zcu);
+
+ const have_min_check, const have_max_check = c: {
+ const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
+ const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
+ const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
+ const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0;
+ break :c .{
+ operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
+ dest_pos_bits < operand_pos_bits,
+ };
+ };
+
+ // The worst-case scenario in terms of total instructions and total condbrs is the case where
+ // the result type is an exhaustive enum whose tag type is smaller than the operand type:
+ //
+ // %x = block({
+ // %1 = cmp_lt(%y, @min_allowed_int)
+ // %2 = cmp_gt(%y, @max_allowed_int)
+ // %3 = bool_or(%1, %2)
+ // %4 = cond_br(%3, {
+ // %5 = call(@panic.invalidEnumValue, [])
+ // %6 = unreach()
+ // }, {
+ // %7 = intcast(@res_ty, %y)
+ // %8 = is_named_enum_value(%7)
+ // %9 = cond_br(%8, {
+ // %10 = br(%x, %7)
+ // }, {
+ // %11 = call(@panic.invalidEnumValue, [])
+ // %12 = unreach()
+ // })
+ // })
+ // })
+ //
+ // Note that vectors of enums don't exist -- the worst case for vectors is this:
+ //
+ // %x = block({
+ // %1 = cmp_lt(%y, @min_allowed_int)
+ // %2 = cmp_gt(%y, @max_allowed_int)
+ // %3 = bool_or(%1, %2)
+ // %4 = reduce(%3, .@"or")
+ // %5 = cond_br(%4, {
+ // %6 = call(@panic.invalidEnumValue, [])
+ // %7 = unreach()
+ // }, {
+ // %8 = intcast(@res_ty, %y)
+ // %9 = br(%x, %8)
+ // })
+ // })
+
+ try l.air_instructions.ensureUnusedCapacity(gpa, 12);
+ var body_inst_buf: [12]Air.Inst.Index = undefined;
+ var condbr_buf: [2]CondBr = undefined;
+ var condbr_idx: usize = 0;
+
+ var main_block: Block = .init(&body_inst_buf);
+ var cur_block: *Block = &main_block;
+
+ const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .cast_truncated_data;
+
+ if (have_min_check or have_max_check) {
+ const dest_int_ty = if (dest_is_enum) dest_ty.intTagType(zcu) else dest_ty;
+ const condbr = &condbr_buf[condbr_idx];
+ condbr_idx += 1;
+ const below_min_inst: Air.Inst.Index = if (have_min_check) inst: {
+ const min_val_ref = Air.internedToRef((try dest_int_ty.minInt(pt, operand_ty)).toIntern());
+ break :inst try cur_block.addCmp(l, is_vector, .lt, operand_ref, min_val_ref);
+ } else undefined;
+ const above_max_inst: Air.Inst.Index = if (have_max_check) inst: {
+ const max_val_ref = Air.internedToRef((try dest_int_ty.maxInt(pt, operand_ty)).toIntern());
+ break :inst try cur_block.addCmp(l, is_vector, .gt, operand_ref, max_val_ref);
+ } else undefined;
+ const out_of_range_inst: Air.Inst.Index = inst: {
+ if (have_min_check and have_max_check) break :inst cur_block.add(l, .{
+ .tag = .bool_or,
+ .data = .{ .bin_op = .{
+ .lhs = below_min_inst.toRef(),
+ .rhs = above_max_inst.toRef(),
+ } },
+ });
+ if (have_min_check) break :inst below_min_inst;
+ if (have_max_check) break :inst above_max_inst;
+ unreachable;
+ };
+ const scalar_out_of_range_inst: Air.Inst.Index = if (is_vector) cur_block.add(l, .{
+ .tag = .reduce,
+ .data = .{ .reduce = .{
+ .operand = out_of_range_inst.toRef(),
+ .operation = .Or,
+ } },
+ }) else out_of_range_inst;
+ condbr.* = .init(l, scalar_out_of_range_inst.toRef(), cur_block, .{
+ .true = .cold,
+ .false = .none,
+ });
+ condbr.then_block = .init(cur_block.stealRemainingCapacity());
+ try condbr.then_block.addPanic(l, panic_id);
+ condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
+ cur_block = &condbr.else_block;
+ }
+
+ // Now we know we're in-range, we can intcast:
+ const cast_inst = cur_block.add(l, .{
+ .tag = .intcast,
+ .data = .{ .ty_op = .{
+ .ty = Air.internedToRef(dest_ty.toIntern()),
+ .operand = operand_ref,
+ } },
+ });
+ // For ints we're already done, but for exhaustive enums we must check this is a valid tag.
+ if (dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu) and zcu.backendSupportsFeature(.is_named_enum_value)) {
+ assert(!is_vector); // vectors of enums don't exist
+ // We are building this:
+ // %1 = is_named_enum_value(%cast_inst)
+ // %2 = cond_br(%1, {
+ // <new cursor>
+ // }, {
+ // <panic>
+ // })
+ const is_named_inst = cur_block.add(l, .{
+ .tag = .is_named_enum_value,
+ .data = .{ .un_op = cast_inst.toRef() },
+ });
+ const condbr = &condbr_buf[condbr_idx];
+ condbr_idx += 1;
+ condbr.* = .init(l, is_named_inst.toRef(), cur_block, .{
+ .true = .none,
+ .false = .cold,
+ });
+ condbr.else_block = .init(cur_block.stealRemainingCapacity());
+ try condbr.else_block.addPanic(l, panic_id);
+ condbr.then_block = .init(condbr.else_block.stealRemainingCapacity());
+ cur_block = &condbr.then_block;
+ }
+ // Finally, just `br` to our outer `block`.
+ _ = cur_block.add(l, .{
+ .tag = .br,
+ .data = .{ .br = .{
+ .block_inst = orig_inst,
+ .operand = cast_inst.toRef(),
+ } },
+ });
+ // We might not have used all of the instructions; that's intentional.
+ _ = cur_block.stealRemainingCapacity();
+
+ for (condbr_buf[0..condbr_idx]) |*condbr| try condbr.finish(l);
+ return .{ .ty_pl = .{
+ .ty = Air.internedToRef(dest_ty.toIntern()),
+ .payload = try l.addBlockBody(main_block.body()),
+ } };
+}
+fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_op_tag: Air.Inst.Tag) Error!Air.Inst.Data {
+ const pt = l.pt;
+ const zcu = pt.zcu;
+ const gpa = zcu.gpa;
+ const bin_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].bin_op;
+
+ const operand_ty = l.typeOf(bin_op.lhs);
+ assert(l.typeOf(bin_op.rhs).toIntern() == operand_ty.toIntern());
+ const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
+
+ const overflow_tuple_ty = try pt.overflowArithmeticTupleType(operand_ty);
+ const overflow_bits_ty = overflow_tuple_ty.fieldType(1, zcu);
+
+ // The worst-case scenario is a vector operand:
+ //
+ // %1 = add_with_overflow(%x, %y)
+ // %2 = struct_field_val(%1, .@"1")
+ // %3 = reduce(%2, .@"or")
+ // %4 = bitcast(%3, @bool_type)
+ // %5 = cond_br(%4, {
+ // %6 = call(@panic.integerOverflow, [])
+ // %7 = unreach()
+ // }, {
+ // %8 = struct_field_val(%1, .@"0")
+ // %9 = br(%z, %8)
+ // })
+ try l.air_instructions.ensureUnusedCapacity(gpa, 9);
+ var body_inst_buf: [9]Air.Inst.Index = undefined;
+
+ var main_block: Block = .init(&body_inst_buf);
+
+ const overflow_op_inst = main_block.add(l, .{
+ .tag = overflow_op_tag,
+ .data = .{ .ty_pl = .{
+ .ty = Air.internedToRef(overflow_tuple_ty.toIntern()),
+ .payload = try l.addExtra(Air.Bin, .{
+ .lhs = bin_op.lhs,
+ .rhs = bin_op.rhs,
+ }),
+ } },
+ });
+ const overflow_bits_inst = main_block.add(l, .{
+ .tag = .struct_field_val,
+ .data = .{ .ty_pl = .{
+ .ty = Air.internedToRef(overflow_bits_ty.toIntern()),
+ .payload = try l.addExtra(Air.StructField, .{
+ .struct_operand = overflow_op_inst.toRef(),
+ .field_index = 1,
+ }),
+ } },
+ });
+ const any_overflow_bit_inst = if (is_vector) main_block.add(l, .{
+ .tag = .reduce,
+ .data = .{ .reduce = .{
+ .operand = overflow_bits_inst.toRef(),
+ .operation = .Or,
+ } },
+ }) else overflow_bits_inst;
+ const any_overflow_inst = try main_block.addCmp(l, false, .eq, any_overflow_bit_inst.toRef(), .one_u1);
+
+ var condbr: CondBr = .init(l, any_overflow_inst.toRef(), &main_block, .{
+ .true = .cold,
+ .false = .none,
+ });
+ condbr.then_block = .init(main_block.stealRemainingCapacity());
+ try condbr.then_block.addPanic(l, .integer_overflow);
+ condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
+
+ const result_inst = condbr.else_block.add(l, .{
+ .tag = .struct_field_val,
+ .data = .{ .ty_pl = .{
+ .ty = Air.internedToRef(operand_ty.toIntern()),
+ .payload = try l.addExtra(Air.StructField, .{
+ .struct_operand = overflow_op_inst.toRef(),
+ .field_index = 0,
+ }),
+ } },
+ });
+ _ = condbr.else_block.add(l, .{
+ .tag = .br,
+ .data = .{ .br = .{
+ .block_inst = orig_inst,
+ .operand = result_inst.toRef(),
+ } },
+ });
+ // We might not have used all of the instructions; that's intentional.
+ _ = condbr.else_block.stealRemainingCapacity();
+
+ try condbr.finish(l);
+ return .{ .ty_pl = .{
+ .ty = Air.internedToRef(operand_ty.toIntern()),
+ .payload = try l.addBlockBody(main_block.body()),
+ } };
+}
-fn Block(comptime capacity: usize) type {
- return struct {
- instructions: [capacity]Air.Inst.Index,
- len: usize,
+const Block = struct {
+ instructions: []Air.Inst.Index,
+ len: usize,
- const empty: @This() = .{
- .instructions = undefined,
+ /// There are two common usages of the API:
+ /// * `buf.len` is exactly the number of instructions which will be in this block
+ /// * `buf.len` is no smaller than necessary, and `b.stealRemainingCapacity` will be used
+ fn init(buf: []Air.Inst.Index) Block {
+ return .{
+ .instructions = buf,
.len = 0,
};
+ }
- fn add(b: *@This(), inst: Air.Inst.Index) Air.Inst.Index {
- b.instructions[b.len] = inst;
- b.len += 1;
- return inst;
+ /// Like `Legalize.addInstAssumeCapacity`, but also appends the instruction to `b`.
+ fn add(b: *Block, l: *Legalize, inst_data: Air.Inst) Air.Inst.Index {
+ const inst = l.addInstAssumeCapacity(inst_data);
+ b.instructions[b.len] = inst;
+ b.len += 1;
+ return inst;
+ }
+
+ /// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
+ /// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
+ fn addPanic(b: *Block, l: *Legalize, panic_id: Zcu.SimplePanicId) Error!void {
+ const zcu = l.pt.zcu;
+ if (!zcu.backendSupportsFeature(.panic_fn)) {
+ _ = b.add(l, .{
+ .tag = .trap,
+ .data = .{ .no_op = {} },
+ });
+ return;
}
+ const panic_fn_val = zcu.builtin_decl_values.get(panic_id.toBuiltin());
+ _ = b.add(l, .{
+ .tag = .call,
+ .data = .{ .pl_op = .{
+ .operand = Air.internedToRef(panic_fn_val),
+ .payload = try l.addExtra(Air.Call, .{ .args_len = 0 }),
+ } },
+ });
+ _ = b.add(l, .{
+ .tag = .unreach,
+ .data = .{ .no_op = {} },
+ });
+ }
- fn body(b: *const @This()) []const Air.Inst.Index {
- assert(b.len == b.instructions.len);
- return &b.instructions;
+ /// Adds a `cmp_*` instruction (including maybe `cmp_vector`) to `b`. This is a fairly thin wrapper
+ /// around `add`, although it does compute the result type if `is_vector` (`@Vector(n, bool)`).
+ fn addCmp(
+ b: *Block,
+ l: *Legalize,
+ is_vector: bool,
+ op: std.math.CompareOperator,
+ lhs: Air.Inst.Ref,
+ rhs: Air.Inst.Ref,
+ ) Error!Air.Inst.Index {
+ const pt = l.pt;
+ if (is_vector) {
+ const bool_vec_ty = try pt.vectorType(.{
+ .child = .bool_type,
+ .len = l.typeOf(lhs).vectorLen(pt.zcu),
+ });
+ return b.add(l, .{
+ .tag = .cmp_vector,
+ .data = .{ .ty_pl = .{
+ .ty = Air.internedToRef(bool_vec_ty.toIntern()),
+ .payload = try l.addExtra(Air.VectorCmp, .{
+ .lhs = lhs,
+ .rhs = rhs,
+ .op = Air.VectorCmp.encodeOp(op),
+ }),
+ } },
+ });
}
+ return b.add(l, .{
+ .tag = switch (op) {
+ .lt => .cmp_lt,
+ .lte => .cmp_lte,
+ .eq => .cmp_eq,
+ .gte => .cmp_gte,
+ .gt => .cmp_gt,
+ .neq => .cmp_neq,
+ },
+ .data = .{ .bin_op = .{
+ .lhs = lhs,
+ .rhs = rhs,
+ } },
+ });
+ }
+
+ /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`.
+ /// This is useful when you've provided a buffer big enough for all your instructions, but you are
+ /// now starting a new block and some of them need to live there instead.
+ fn stealRemainingCapacity(b: *Block) []Air.Inst.Index {
+ const remaining = b.instructions[b.len..];
+ b.instructions = b.instructions[0..b.len];
+ return remaining;
+ }
+
+ fn body(b: *const Block) []const Air.Inst.Index {
+ assert(b.len == b.instructions.len);
+ return b.instructions;
+ }
+};
+
+const CondBr = struct {
+ inst: Air.Inst.Index,
+ hints: BranchHints,
+ then_block: Block,
+ else_block: Block,
+
+ const BranchHints = struct {
+ true: std.builtin.BranchHint,
+ false: std.builtin.BranchHint,
};
-}
+
+ /// The return value has `then_block` and `else_block` initialized to `undefined`; it is the
+ /// caller's reponsibility to initialize them.
+ fn init(l: *Legalize, operand: Air.Inst.Ref, parent_block: *Block, hints: BranchHints) CondBr {
+ return .{
+ .inst = parent_block.add(l, .{
+ .tag = .cond_br,
+ .data = .{ .pl_op = .{
+ .operand = operand,
+ .payload = undefined,
+ } },
+ }),
+ .hints = hints,
+ .then_block = undefined,
+ .else_block = undefined,
+ };
+ }
+
+ fn finish(cond_br: CondBr, l: *Legalize) Error!void {
+ const data = &l.air_instructions.items(.data)[@intFromEnum(cond_br.inst)];
+ data.pl_op.payload = try l.addCondBrBodiesHints(
+ cond_br.then_block.body(),
+ cond_br.else_block.body(),
+ .{
+ .true = cond_br.hints.true,
+ .false = cond_br.hints.false,
+ .then_cov = .none,
+ .else_cov = .none,
+ },
+ );
+ }
+};
fn addInstAssumeCapacity(l: *Legalize, inst: Air.Inst) Air.Inst.Index {
defer l.air_instructions.appendAssumeCapacity(inst);
@@ -818,17 +1253,20 @@ fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 {
}
fn addCondBrBodies(l: *Legalize, then_body: []const Air.Inst.Index, else_body: []const Air.Inst.Index) Error!u32 {
+ return l.addCondBrBodiesHints(then_body, else_body, .{
+ .true = .none,
+ .false = .none,
+ .then_cov = .none,
+ .else_cov = .none,
+ });
+}
+fn addCondBrBodiesHints(l: *Legalize, then_body: []const Air.Inst.Index, else_body: []const Air.Inst.Index, hints: Air.CondBr.BranchHints) Error!u32 {
try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 3 + then_body.len + else_body.len);
defer {
l.air_extra.appendSliceAssumeCapacity(&.{
@intCast(then_body.len),
@intCast(else_body.len),
- @bitCast(Air.CondBr.BranchHints{
- .true = .none,
- .false = .none,
- .then_cov = .none,
- .else_cov = .none,
- }),
+ @bitCast(hints),
});
l.air_extra.appendSliceAssumeCapacity(@ptrCast(then_body));
l.air_extra.appendSliceAssumeCapacity(@ptrCast(else_body));
src/arch/riscv64/CodeGen.zig
@@ -52,7 +52,12 @@ const Instruction = encoding.Instruction;
const InnerError = CodeGenError || error{OutOfRegisters};
pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
- return null;
+ return comptime &.initMany(&.{
+ .expand_intcast_safe,
+ .expand_add_safe,
+ .expand_sub_safe,
+ .expand_mul_safe,
+ });
}
pt: Zcu.PerThread,
src/arch/wasm/CodeGen.zig
@@ -32,7 +32,12 @@ const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
- return null;
+ return comptime &.initMany(&.{
+ .expand_intcast_safe,
+ .expand_add_safe,
+ .expand_sub_safe,
+ .expand_mul_safe,
+ });
}
/// Reference to the function declaration the code
src/arch/x86_64/CodeGen.zig
@@ -88,6 +88,10 @@ pub fn legalizeFeatures(target: *const std.Target) *const Air.Legalize.Features
.unsplat_shift_rhs = false,
.reduce_one_elem_to_bitcast = true,
+ .expand_intcast_safe = true,
+ .expand_add_safe = true,
+ .expand_sub_safe = true,
+ .expand_mul_safe = true,
}),
};
}
src/codegen/spirv.zig
@@ -29,7 +29,12 @@ const SpvAssembler = @import("spirv/Assembler.zig");
const InstMap = std.AutoHashMapUnmanaged(Air.Inst.Index, IdRef);
pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
- return null;
+ return comptime &.initMany(&.{
+ .expand_intcast_safe,
+ .expand_add_safe,
+ .expand_sub_safe,
+ .expand_mul_safe,
+ });
}
pub const zig_call_abi_ver = 3;
src/Zcu/PerThread.zig
@@ -3844,6 +3844,21 @@ pub fn nullValue(pt: Zcu.PerThread, opt_ty: Type) Allocator.Error!Value {
} }));
}
+/// `ty` is an integer or a vector of integers.
+pub fn overflowArithmeticTupleType(pt: Zcu.PerThread, ty: Type) !Type {
+ const zcu = pt.zcu;
+ const ip = &zcu.intern_pool;
+ const ov_ty: Type = if (ty.zigTypeTag(zcu) == .vector) try pt.vectorType(.{
+ .len = ty.vectorLen(zcu),
+ .child = .u1_type,
+ }) else .u1;
+ const tuple_ty = try ip.getTupleType(zcu.gpa, pt.tid, .{
+ .types = &.{ ty.toIntern(), ov_ty.toIntern() },
+ .values = &.{ .none, .none },
+ });
+ return .fromInterned(tuple_ty);
+}
+
pub fn smallestUnsignedInt(pt: Zcu.PerThread, max: u64) Allocator.Error!Type {
return pt.intType(.unsigned, Type.smallestUnsignedBits(max));
}
src/Air.zig
@@ -50,8 +50,6 @@ pub const Inst = struct {
/// is the same as both operands.
/// The panic handler function must be populated before lowering AIR
/// that contains this instruction.
- /// This instruction will only be emitted if the backend has the
- /// feature `safety_checked_instructions`.
/// Uses the `bin_op` field.
add_safe,
/// Float addition. The instruction is allowed to have equal or more
@@ -79,8 +77,6 @@ pub const Inst = struct {
/// is the same as both operands.
/// The panic handler function must be populated before lowering AIR
/// that contains this instruction.
- /// This instruction will only be emitted if the backend has the
- /// feature `safety_checked_instructions`.
/// Uses the `bin_op` field.
sub_safe,
/// Float subtraction. The instruction is allowed to have equal or more
@@ -108,8 +104,6 @@ pub const Inst = struct {
/// is the same as both operands.
/// The panic handler function must be populated before lowering AIR
/// that contains this instruction.
- /// This instruction will only be emitted if the backend has the
- /// feature `safety_checked_instructions`.
/// Uses the `bin_op` field.
mul_safe,
/// Float multiplication. The instruction is allowed to have equal or more
src/Sema.zig
@@ -8912,21 +8912,10 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
try sema.requireRuntimeBlock(block, src, operand_src);
if (block.wantSafety()) {
- if (zcu.backendSupportsFeature(.safety_checked_instructions)) {
+ if (zcu.backendSupportsFeature(.panic_fn)) {
_ = try sema.preparePanicId(src, .invalid_enum_value);
- return block.addTyOp(.intcast_safe, dest_ty, operand);
- } else {
- // Slightly silly fallback case...
- const int_tag_ty = dest_ty.intTagType(zcu);
- // Use `intCast`, since it'll set up the Sema-emitted safety checks for us!
- const int_val = try sema.intCast(block, src, int_tag_ty, src, operand, src, true, true);
- const result = try block.addBitCast(dest_ty, int_val);
- if (!dest_ty.isNonexhaustiveEnum(zcu) and zcu.backendSupportsFeature(.is_named_enum_value)) {
- const ok = try block.addUnOp(.is_named_enum_value, result);
- try sema.addSafetyCheck(block, src, ok, .invalid_enum_value);
- }
- return result;
}
+ return block.addTyOp(.intcast_safe, dest_ty, operand);
}
return block.addTyOp(.intcast, dest_ty, operand);
}
@@ -10331,90 +10320,11 @@ fn intCast(
try sema.requireRuntimeBlock(block, src, operand_src);
if (runtime_safety and block.wantSafety()) {
- if (zcu.backendSupportsFeature(.safety_checked_instructions)) {
+ if (zcu.backendSupportsFeature(.panic_fn)) {
_ = try sema.preparePanicId(src, .negative_to_unsigned);
_ = try sema.preparePanicId(src, .cast_truncated_data);
- return block.addTyOp(.intcast_safe, dest_ty, operand);
- }
- const actual_info = operand_scalar_ty.intInfo(zcu);
- const wanted_info = dest_scalar_ty.intInfo(zcu);
- const actual_bits = actual_info.bits;
- const wanted_bits = wanted_info.bits;
- const actual_value_bits = actual_bits - @intFromBool(actual_info.signedness == .signed);
- const wanted_value_bits = wanted_bits - @intFromBool(wanted_info.signedness == .signed);
-
- // range shrinkage
- // requirement: int value fits into target type
- if (wanted_value_bits < actual_value_bits) {
- const dest_max_val_scalar = try dest_scalar_ty.maxIntScalar(pt, operand_scalar_ty);
- const dest_max_val = try sema.splat(operand_ty, dest_max_val_scalar);
- const dest_max = Air.internedToRef(dest_max_val.toIntern());
-
- if (actual_info.signedness == .signed) {
- const diff = try block.addBinOp(.sub_wrap, dest_max, operand);
-
- // Reinterpret the sign-bit as part of the value. This will make
- // negative differences (`operand` > `dest_max`) appear too big.
- const unsigned_scalar_operand_ty = try pt.intType(.unsigned, actual_bits);
- const unsigned_operand_ty = if (is_vector) try pt.vectorType(.{
- .len = dest_ty.vectorLen(zcu),
- .child = unsigned_scalar_operand_ty.toIntern(),
- }) else unsigned_scalar_operand_ty;
- const diff_unsigned = try block.addBitCast(unsigned_operand_ty, diff);
-
- // If the destination type is signed, then we need to double its
- // range to account for negative values.
- const dest_range_val = if (wanted_info.signedness == .signed) range_val: {
- const one_scalar = try pt.intValue(unsigned_scalar_operand_ty, 1);
- const one = if (is_vector) Value.fromInterned(try pt.intern(.{ .aggregate = .{
- .ty = unsigned_operand_ty.toIntern(),
- .storage = .{ .repeated_elem = one_scalar.toIntern() },
- } })) else one_scalar;
- const range_minus_one = try dest_max_val.shl(one, unsigned_operand_ty, sema.arena, pt);
- const result = try arith.addWithOverflow(sema, unsigned_operand_ty, range_minus_one, one);
- assert(result.overflow_bit.compareAllWithZero(.eq, zcu));
- break :range_val result.wrapped_result;
- } else try pt.getCoerced(dest_max_val, unsigned_operand_ty);
- const dest_range = Air.internedToRef(dest_range_val.toIntern());
-
- const ok = if (is_vector) ok: {
- const is_in_range = try block.addCmpVector(diff_unsigned, dest_range, .lte);
- const all_in_range = try block.addReduce(is_in_range, .And);
- break :ok all_in_range;
- } else ok: {
- const is_in_range = try block.addBinOp(.cmp_lte, diff_unsigned, dest_range);
- break :ok is_in_range;
- };
- // TODO negative_to_unsigned?
- try sema.addSafetyCheck(block, src, ok, if (safety_panics_are_enum) .invalid_enum_value else .cast_truncated_data);
- } else {
- const ok = if (is_vector) ok: {
- const is_in_range = try block.addCmpVector(operand, dest_max, .lte);
- const all_in_range = try block.addReduce(is_in_range, .And);
- break :ok all_in_range;
- } else ok: {
- const is_in_range = try block.addBinOp(.cmp_lte, operand, dest_max);
- break :ok is_in_range;
- };
- try sema.addSafetyCheck(block, src, ok, if (safety_panics_are_enum) .invalid_enum_value else .cast_truncated_data);
- }
- } else if (actual_info.signedness == .signed and wanted_info.signedness == .unsigned) {
- // no shrinkage, yes sign loss
- // requirement: signed to unsigned >= 0
- const ok = if (is_vector) ok: {
- const scalar_zero = try pt.intValue(operand_scalar_ty, 0);
- const zero_val = try sema.splat(operand_ty, scalar_zero);
- const zero_inst = Air.internedToRef(zero_val.toIntern());
- const is_in_range = try block.addCmpVector(operand, zero_inst, .gte);
- const all_in_range = try block.addReduce(is_in_range, .And);
- break :ok all_in_range;
- } else ok: {
- const zero_inst = Air.internedToRef((try pt.intValue(operand_ty, 0)).toIntern());
- const is_in_range = try block.addBinOp(.cmp_gte, operand, zero_inst);
- break :ok is_in_range;
- };
- try sema.addSafetyCheck(block, src, ok, if (safety_panics_are_enum) .invalid_enum_value else .negative_to_unsigned);
}
+ return block.addTyOp(.intcast_safe, dest_ty, operand);
}
return block.addTyOp(.intcast, dest_ty, operand);
}
@@ -14316,7 +14226,7 @@ fn zirShl(
}
if (air_tag == .shl_exact) {
- const op_ov_tuple_ty = try sema.overflowArithmeticTupleType(lhs_ty);
+ const op_ov_tuple_ty = try pt.overflowArithmeticTupleType(lhs_ty);
const op_ov = try block.addInst(.{
.tag = .shl_with_overflow,
.data = .{ .ty_pl = .{
@@ -16111,7 +16021,7 @@ fn zirOverflowArithmetic(
const maybe_lhs_val = try sema.resolveValue(lhs);
const maybe_rhs_val = try sema.resolveValue(rhs);
- const tuple_ty = try sema.overflowArithmeticTupleType(dest_ty);
+ const tuple_ty = try pt.overflowArithmeticTupleType(dest_ty);
const overflow_ty: Type = .fromInterned(ip.indexToKey(tuple_ty.toIntern()).tuple_type.types.get(ip)[1]);
var result: struct {
@@ -16284,24 +16194,6 @@ fn splat(sema: *Sema, ty: Type, val: Value) !Value {
return Value.fromInterned(repeated);
}
-fn overflowArithmeticTupleType(sema: *Sema, ty: Type) !Type {
- const pt = sema.pt;
- const zcu = pt.zcu;
- const ip = &zcu.intern_pool;
- const ov_ty: Type = if (ty.zigTypeTag(zcu) == .vector) try pt.vectorType(.{
- .len = ty.vectorLen(zcu),
- .child = .u1_type,
- }) else .u1;
-
- const types = [2]InternPool.Index{ ty.toIntern(), ov_ty.toIntern() };
- const values = [2]InternPool.Index{ .none, .none };
- const tuple_ty = try ip.getTupleType(zcu.gpa, pt.tid, .{
- .types = &types,
- .values = &values,
- });
- return .fromInterned(tuple_ty);
-}
-
fn analyzeArithmetic(
sema: *Sema,
block: *Block,
@@ -16477,41 +16369,10 @@ fn analyzeArithmetic(
}
if (block.wantSafety() and want_safety and scalar_tag == .int) {
- if (zcu.backendSupportsFeature(.safety_checked_instructions)) {
- if (air_tag != air_tag_safe) {
- _ = try sema.preparePanicId(src, .integer_overflow);
- }
- return block.addBinOp(air_tag_safe, casted_lhs, casted_rhs);
- } else {
- const maybe_op_ov: ?Air.Inst.Tag = switch (air_tag) {
- .add => .add_with_overflow,
- .sub => .sub_with_overflow,
- .mul => .mul_with_overflow,
- else => null,
- };
- if (maybe_op_ov) |op_ov_tag| {
- const op_ov_tuple_ty = try sema.overflowArithmeticTupleType(resolved_type);
- const op_ov = try block.addInst(.{
- .tag = op_ov_tag,
- .data = .{ .ty_pl = .{
- .ty = Air.internedToRef(op_ov_tuple_ty.toIntern()),
- .payload = try sema.addExtra(Air.Bin{
- .lhs = casted_lhs,
- .rhs = casted_rhs,
- }),
- } },
- });
- const ov_bit = try sema.tupleFieldValByIndex(block, op_ov, 1, op_ov_tuple_ty);
- const any_ov_bit = if (resolved_type.zigTypeTag(zcu) == .vector)
- try block.addReduce(ov_bit, .Or)
- else
- ov_bit;
- const no_ov = try block.addBinOp(.cmp_eq, any_ov_bit, .zero_u1);
-
- try sema.addSafetyCheck(block, src, no_ov, .integer_overflow);
- return sema.tupleFieldValByIndex(block, op_ov, 0, op_ov_tuple_ty);
- }
+ if (air_tag != air_tag_safe and zcu.backendSupportsFeature(.panic_fn)) {
+ _ = try sema.preparePanicId(src, .integer_overflow);
}
+ return block.addBinOp(air_tag_safe, casted_lhs, casted_rhs);
}
return block.addBinOp(air_tag, casted_lhs, casted_rhs);
}
src/target.zig
@@ -842,10 +842,6 @@ pub inline fn backendSupportsFeature(backend: std.builtin.CompilerBackend, compt
.stage2_c, .stage2_llvm, .stage2_x86_64 => true,
else => false,
},
- .safety_checked_instructions => switch (backend) {
- .stage2_llvm => true,
- else => false,
- },
.separate_thread => switch (backend) {
.stage2_llvm => false,
else => true,
src/Zcu.zig
@@ -3829,15 +3829,6 @@ pub const Feature = enum {
is_named_enum_value,
error_set_has_value,
field_reordering,
- /// When this feature is supported, the backend supports the following AIR instructions:
- /// * `Air.Inst.Tag.add_safe`
- /// * `Air.Inst.Tag.sub_safe`
- /// * `Air.Inst.Tag.mul_safe`
- /// * `Air.Inst.Tag.intcast_safe`
- /// The motivation for this feature is that it makes AIR smaller, and makes it easier
- /// to generate better machine code in the backends. All backends should migrate to
- /// enabling this feature.
- safety_checked_instructions,
/// If the backend supports running from another thread.
separate_thread,
};