Commit 00878a15d7
Changed files (5)
lib
src-self-hosted
lib/std/zig/ast.zig
@@ -137,7 +137,6 @@ pub const Error = union(enum) {
ExpectedCallOrFnProto: ExpectedCallOrFnProto,
ExpectedSliceOrRBracket: ExpectedSliceOrRBracket,
ExtraAlignQualifier: ExtraAlignQualifier,
- ExtraNullQualifier: ExtraNullQualifier,
ExtraConstQualifier: ExtraConstQualifier,
ExtraVolatileQualifier: ExtraVolatileQualifier,
ExtraAllowZeroQualifier: ExtraAllowZeroQualifier,
@@ -185,7 +184,6 @@ pub const Error = union(enum) {
.ExpectedCallOrFnProto => |*x| return x.render(tokens, stream),
.ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream),
.ExtraAlignQualifier => |*x| return x.render(tokens, stream),
- .ExtraNullQualifier => |*x| return x.render(tokens, stream),
.ExtraConstQualifier => |*x| return x.render(tokens, stream),
.ExtraVolatileQualifier => |*x| return x.render(tokens, stream),
.ExtraAllowZeroQualifier => |*x| return x.render(tokens, stream),
@@ -235,7 +233,6 @@ pub const Error = union(enum) {
.ExpectedCallOrFnProto => |x| return x.node.firstToken(),
.ExpectedSliceOrRBracket => |x| return x.token,
.ExtraAlignQualifier => |x| return x.token,
- .ExtraNullQualifier => |x| return x.token,
.ExtraConstQualifier => |x| return x.token,
.ExtraVolatileQualifier => |x| return x.token,
.ExtraAllowZeroQualifier => |x| return x.token,
@@ -296,7 +293,6 @@ pub const Error = union(enum) {
pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub");
pub const UnattachedDocComment = SimpleError("Unattached documentation comment");
pub const ExtraAlignQualifier = SimpleError("Extra align qualifier");
- pub const ExtraNullQualifier = SimpleError("Extra null qualifier");
pub const ExtraConstQualifier = SimpleError("Extra const qualifier");
pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");
pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
@@ -1535,7 +1531,7 @@ pub const Node = struct {
};
pub const PrefixOp = struct {
- base: Node,
+ base: Node = Node{ .id = .PrefixOp },
op_token: TokenIndex,
op: Op,
rhs: *Node,
@@ -1558,15 +1554,15 @@ pub const Node = struct {
pub const ArrayInfo = struct {
len_expr: *Node,
- null_token: ?TokenIndex,
+ sentinel: ?*Node,
};
pub const PtrInfo = struct {
- allowzero_token: ?TokenIndex,
- align_info: ?Align,
- const_token: ?TokenIndex,
- volatile_token: ?TokenIndex,
- null_token: ?TokenIndex,
+ allowzero_token: ?TokenIndex = null,
+ align_info: ?Align = null,
+ const_token: ?TokenIndex = null,
+ volatile_token: ?TokenIndex = null,
+ sentinel: ?*Node = null,
pub const Align = struct {
node: *Node,
@@ -1585,6 +1581,11 @@ pub const Node = struct {
switch (self.op) {
// TODO https://github.com/ziglang/zig/issues/1107
Op.SliceType => |addr_of_info| {
+ if (addr_of_info.sentinel) |sentinel| {
+ if (i < 1) return sentinel;
+ i -= 1;
+ }
+
if (addr_of_info.align_info) |align_info| {
if (i < 1) return align_info.node;
i -= 1;
@@ -1601,6 +1602,10 @@ pub const Node = struct {
Op.ArrayType => |array_info| {
if (i < 1) return array_info.len_expr;
i -= 1;
+ if (array_info.sentinel) |sentinel| {
+ if (i < 1) return sentinel;
+ i -= 1;
+ }
},
Op.AddressOf,
lib/std/zig/parse.zig
@@ -2246,73 +2246,6 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
return &node.base;
}
- if (try parseArrayTypeStart(arena, it, tree)) |node| {
- switch (node.cast(Node.PrefixOp).?.op) {
- .ArrayType => {},
- .SliceType => |*slice_type| {
- // Collect pointer qualifiers in any order, but disallow duplicates
- while (true) {
- if (eatToken(it, .Keyword_null)) |null_token| {
- if (slice_type.null_token != null) {
- try tree.errors.push(AstError{
- .ExtraNullQualifier = AstError.ExtraNullQualifier{ .token = it.index },
- });
- return error.ParseError;
- }
- slice_type.null_token = null_token;
- continue;
- }
- if (try parseByteAlign(arena, it, tree)) |align_expr| {
- if (slice_type.align_info != null) {
- try tree.errors.push(AstError{
- .ExtraAlignQualifier = AstError.ExtraAlignQualifier{ .token = it.index },
- });
- return error.ParseError;
- }
- slice_type.align_info = Node.PrefixOp.PtrInfo.Align{
- .node = align_expr,
- .bit_range = null,
- };
- continue;
- }
- if (eatToken(it, .Keyword_const)) |const_token| {
- if (slice_type.const_token != null) {
- try tree.errors.push(AstError{
- .ExtraConstQualifier = AstError.ExtraConstQualifier{ .token = it.index },
- });
- return error.ParseError;
- }
- slice_type.const_token = const_token;
- continue;
- }
- if (eatToken(it, .Keyword_volatile)) |volatile_token| {
- if (slice_type.volatile_token != null) {
- try tree.errors.push(AstError{
- .ExtraVolatileQualifier = AstError.ExtraVolatileQualifier{ .token = it.index },
- });
- return error.ParseError;
- }
- slice_type.volatile_token = volatile_token;
- continue;
- }
- if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
- if (slice_type.allowzero_token != null) {
- try tree.errors.push(AstError{
- .ExtraAllowZeroQualifier = AstError.ExtraAllowZeroQualifier{ .token = it.index },
- });
- return error.ParseError;
- }
- slice_type.allowzero_token = allowzero_token;
- continue;
- }
- break;
- }
- },
- else => unreachable,
- }
- return node;
- }
-
if (try parsePtrTypeStart(arena, it, tree)) |node| {
// If the token encountered was **, there will be two nodes instead of one.
// The attributes should be applied to the rightmost operator.
@@ -2323,10 +2256,6 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
&prefix_op.op.PtrType;
while (true) {
- if (eatToken(it, .Keyword_null)) |null_token| {
- ptr_info.null_token = null_token;
- continue;
- }
if (eatToken(it, .Keyword_align)) |align_token| {
const lparen = try expectToken(it, tree, .LParen);
const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{
@@ -2375,6 +2304,63 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
return node;
}
+ if (try parseArrayTypeStart(arena, it, tree)) |node| {
+ switch (node.cast(Node.PrefixOp).?.op) {
+ .ArrayType => {},
+ .SliceType => |*slice_type| {
+ // Collect pointer qualifiers in any order, but disallow duplicates
+ while (true) {
+ if (try parseByteAlign(arena, it, tree)) |align_expr| {
+ if (slice_type.align_info != null) {
+ try tree.errors.push(AstError{
+ .ExtraAlignQualifier = AstError.ExtraAlignQualifier{ .token = it.index },
+ });
+ return error.ParseError;
+ }
+ slice_type.align_info = Node.PrefixOp.PtrInfo.Align{
+ .node = align_expr,
+ .bit_range = null,
+ };
+ continue;
+ }
+ if (eatToken(it, .Keyword_const)) |const_token| {
+ if (slice_type.const_token != null) {
+ try tree.errors.push(AstError{
+ .ExtraConstQualifier = AstError.ExtraConstQualifier{ .token = it.index },
+ });
+ return error.ParseError;
+ }
+ slice_type.const_token = const_token;
+ continue;
+ }
+ if (eatToken(it, .Keyword_volatile)) |volatile_token| {
+ if (slice_type.volatile_token != null) {
+ try tree.errors.push(AstError{
+ .ExtraVolatileQualifier = AstError.ExtraVolatileQualifier{ .token = it.index },
+ });
+ return error.ParseError;
+ }
+ slice_type.volatile_token = volatile_token;
+ continue;
+ }
+ if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
+ if (slice_type.allowzero_token != null) {
+ try tree.errors.push(AstError{
+ .ExtraAllowZeroQualifier = AstError.ExtraAllowZeroQualifier{ .token = it.index },
+ });
+ return error.ParseError;
+ }
+ slice_type.allowzero_token = allowzero_token;
+ continue;
+ }
+ break;
+ }
+ },
+ else => unreachable,
+ }
+ return node;
+ }
+
return null;
}
@@ -2473,14 +2459,19 @@ const AnnotatedParamList = struct {
fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
const lbracket = eatToken(it, .LBracket) orelse return null;
const expr = try parseExpr(arena, it, tree);
+ const sentinel = if (eatToken(it, .Colon)) |_|
+ try expectNode(arena, it, tree, parseExpr, AstError{
+ .ExpectedExpr = .{ .token = it.index },
+ })
+ else
+ null;
const rbracket = try expectToken(it, tree, .RBracket);
- const null_token = eatToken(it, .Keyword_null);
const op = if (expr) |len_expr|
Node.PrefixOp.Op{
.ArrayType = .{
.len_expr = len_expr,
- .null_token = null_token,
+ .sentinel = sentinel,
},
}
else
@@ -2490,7 +2481,7 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
.align_info = null,
.const_token = null,
.volatile_token = null,
- .null_token = null,
+ .sentinel = sentinel,
},
};
@@ -2510,49 +2501,76 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
/// / PTRUNKNOWN
/// / PTRC
fn parsePtrTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
- const token = eatAnnotatedToken(it, .Asterisk) orelse
- eatAnnotatedToken(it, .AsteriskAsterisk) orelse
- eatAnnotatedToken(it, .BracketStarBracket) orelse
- eatAnnotatedToken(it, .BracketStarCBracket) orelse
- return null;
+ if (eatToken(it, .Asterisk)) |asterisk| {
+ const sentinel = if (eatToken(it, .Colon)) |_|
+ try expectNode(arena, it, tree, parseExpr, AstError{
+ .ExpectedExpr = .{ .token = it.index },
+ })
+ else
+ null;
+ const node = try arena.create(Node.PrefixOp);
+ node.* = .{
+ .op_token = asterisk,
+ .op = .{ .PtrType = .{ .sentinel = sentinel } },
+ .rhs = undefined, // set by caller
+ };
+ return &node.base;
+ }
- const node = try arena.create(Node.PrefixOp);
- node.* = Node.PrefixOp{
- .base = Node{ .id = .PrefixOp },
- .op_token = token.index,
- .op = Node.PrefixOp.Op{
- .PtrType = Node.PrefixOp.PtrInfo{
- .allowzero_token = null,
- .align_info = null,
- .const_token = null,
- .volatile_token = null,
- .null_token = null,
- },
- },
- .rhs = undefined, // set by caller
- };
+ if (eatToken(it, .AsteriskAsterisk)) |double_asterisk| {
+ const node = try arena.create(Node.PrefixOp);
+ node.* = Node.PrefixOp{
+ .op_token = double_asterisk,
+ .op = Node.PrefixOp.Op{ .PtrType = .{} },
+ .rhs = undefined, // set by caller
+ };
- // Special case for **, which is its own token
- if (token.ptr.id == .AsteriskAsterisk) {
+ // Special case for **, which is its own token
const child = try arena.create(Node.PrefixOp);
child.* = Node.PrefixOp{
- .base = Node{ .id = .PrefixOp },
- .op_token = token.index,
- .op = Node.PrefixOp.Op{
- .PtrType = Node.PrefixOp.PtrInfo{
- .allowzero_token = null,
- .align_info = null,
- .const_token = null,
- .volatile_token = null,
- .null_token = null,
- },
- },
+ .op_token = double_asterisk,
+ .op = Node.PrefixOp.Op{ .PtrType = .{} },
.rhs = undefined, // set by caller
};
node.rhs = &child.base;
- }
- return &node.base;
+ return &node.base;
+ }
+ if (eatToken(it, .LBracket)) |lbracket| {
+ const asterisk = eatToken(it, .Asterisk) orelse {
+ putBackToken(it, lbracket);
+ return null;
+ };
+ if (eatToken(it, .Identifier)) |ident| {
+ if (!std.mem.eql(u8, tree.tokenSlice(ident), "c")) {
+ putBackToken(it, ident);
+ } else {
+ _ = try expectToken(it, tree, .RBracket);
+ const node = try arena.create(Node.PrefixOp);
+ node.* = .{
+ .op_token = ident,
+ .op = .{ .PtrType = .{} },
+ .rhs = undefined, // set by caller
+ };
+ return &node.base;
+ }
+ }
+ const sentinel = if (eatToken(it, .Colon)) |_|
+ try expectNode(arena, it, tree, parseExpr, AstError{
+ .ExpectedExpr = .{ .token = it.index },
+ })
+ else
+ null;
+ _ = try expectToken(it, tree, .RBracket);
+ const node = try arena.create(Node.PrefixOp);
+ node.* = .{
+ .op_token = lbracket,
+ .op = .{ .PtrType = .{ .sentinel = sentinel } },
+ .rhs = undefined, // set by caller
+ };
+ return &node.base;
+ }
+ return null;
}
/// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
lib/std/zig/render.zig
@@ -418,13 +418,26 @@ fn renderExpression(
switch (prefix_op_node.op) {
ast.Node.PrefixOp.Op.PtrType => |ptr_info| {
- const star_offset = switch (tree.tokens.at(prefix_op_node.op_token).id) {
- Token.Id.AsteriskAsterisk => @as(usize, 1),
- else => @as(usize, 0),
- };
- try renderTokenOffset(tree, stream, prefix_op_node.op_token, indent, start_col, Space.None, star_offset); // *
- if (ptr_info.null_token) |null_token| {
- try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null
+ const op_tok_id = tree.tokens.at(prefix_op_node.op_token).id;
+ switch (op_tok_id) {
+ .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
+ .Identifier => try stream.write("[*c]"),
+ .LBracket => try stream.write("[*"),
+ else => unreachable,
+ }
+ if (ptr_info.sentinel) |sentinel| {
+ const colon_token = tree.prevToken(sentinel.firstToken());
+ try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // :
+ const sentinel_space = switch (op_tok_id) {
+ .LBracket => Space.None,
+ else => Space.Space,
+ };
+ try renderExpression(allocator, stream, tree, indent, start_col, sentinel, sentinel_space);
+ }
+ switch (op_tok_id) {
+ .Asterisk, .AsteriskAsterisk, .Identifier => {},
+ .LBracket => try stream.writeByte(']'),
+ else => unreachable,
}
if (ptr_info.allowzero_token) |allowzero_token| {
try renderToken(tree, stream, allowzero_token, indent, start_col, Space.Space); // allowzero
@@ -504,7 +517,10 @@ fn renderExpression(
ast.Node.PrefixOp.Op.ArrayType => |array_info| {
const lbracket = prefix_op_node.op_token;
- const rbracket = tree.nextToken(array_info.len_expr.lastToken());
+ const rbracket = tree.nextToken(if (array_info.sentinel) |sentinel|
+ sentinel.lastToken()
+ else
+ array_info.len_expr.lastToken());
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
@@ -519,10 +535,12 @@ fn renderExpression(
if (ends_with_comment or starts_with_comment) {
try stream.writeByteNTimes(' ', indent);
}
- try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ]
- if (array_info.null_token) |null_token| {
- try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null
+ if (array_info.sentinel) |sentinel| {
+ const colon_token = tree.prevToken(sentinel.firstToken());
+ try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // :
+ try renderExpression(allocator, stream, tree, indent, start_col, sentinel, Space.None);
}
+ try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ]
},
ast.Node.PrefixOp.Op.BitNot,
ast.Node.PrefixOp.Op.BoolNot,
lib/std/zig/tokenizer.zig
@@ -143,8 +143,6 @@ pub const Token = struct {
LineComment,
DocComment,
ContainerDocComment,
- BracketStarBracket,
- BracketStarCBracket,
ShebangLine,
Keyword_align,
Keyword_allowzero,
@@ -269,8 +267,6 @@ pub const Token = struct {
.AngleBracketAngleBracketRight => ">>",
.AngleBracketAngleBracketRightEqual => ">>=",
.Tilde => "~",
- .BracketStarBracket => "[*]",
- .BracketStarCBracket => "[*c]",
.Keyword_align => "align",
.Keyword_allowzero => "allowzero",
.Keyword_and => "and",
@@ -400,9 +396,6 @@ pub const Tokenizer = struct {
Period,
Period2,
SawAtSign,
- LBracket,
- LBracketStar,
- LBracketStarC,
};
pub fn next(self: *Tokenizer) Token {
@@ -460,7 +453,9 @@ pub const Tokenizer = struct {
break;
},
'[' => {
- state = State.LBracket;
+ result.id = .LBracket;
+ self.index += 1;
+ break;
},
']' => {
result.id = Token.Id.RBracket;
@@ -564,43 +559,6 @@ pub const Tokenizer = struct {
},
},
- State.LBracket => switch (c) {
- '*' => {
- state = State.LBracketStar;
- },
- else => {
- result.id = Token.Id.LBracket;
- break;
- },
- },
-
- State.LBracketStar => switch (c) {
- 'c' => {
- state = State.LBracketStarC;
- },
- ']' => {
- result.id = Token.Id.BracketStarBracket;
- self.index += 1;
- break;
- },
- else => {
- result.id = Token.Id.Invalid;
- break;
- },
- },
-
- State.LBracketStarC => switch (c) {
- ']' => {
- result.id = Token.Id.BracketStarCBracket;
- self.index += 1;
- break;
- },
- else => {
- result.id = Token.Id.Invalid;
- break;
- },
- },
-
State.Ampersand => switch (c) {
'&' => {
result.id = Token.Id.Invalid_ampersands;
@@ -1227,8 +1185,6 @@ pub const Tokenizer = struct {
State.CharLiteralEnd,
State.CharLiteralUnicode,
State.StringLiteralBackslash,
- State.LBracketStar,
- State.LBracketStarC,
=> {
result.id = Token.Id.Invalid;
},
@@ -1245,9 +1201,6 @@ pub const Tokenizer = struct {
State.Slash => {
result.id = Token.Id.Slash;
},
- State.LBracket => {
- result.id = Token.Id.LBracket;
- },
State.Zero => {
result.id = Token.Id.IntegerLiteral;
},
@@ -1368,9 +1321,14 @@ test "tokenizer - unknown length pointer and then c pointer" {
\\[*]u8
\\[*c]u8
, [_]Token.Id{
- Token.Id.BracketStarBracket,
+ Token.Id.LBracket,
+ Token.Id.Asterisk,
+ Token.Id.RBracket,
+ Token.Id.Identifier,
+ Token.Id.LBracket,
+ Token.Id.Asterisk,
Token.Id.Identifier,
- Token.Id.BracketStarCBracket,
+ Token.Id.RBracket,
Token.Id.Identifier,
});
}
src-self-hosted/translate_c.zig
@@ -1105,19 +1105,32 @@ fn transCreateNodePtrType(
is_const: bool,
is_volatile: bool,
op_tok_id: std.zig.Token.Id,
- bytes: []const u8,
) !*ast.Node.PrefixOp {
const node = try c.a().create(ast.Node.PrefixOp);
+ const op_token = switch (op_tok_id) {
+ .LBracket => blk: {
+ const lbracket = try appendToken(c, .LBracket, "[");
+ _ = try appendToken(c, .Asterisk, "*");
+ _ = try appendToken(c, .RBracket, "]");
+ break :blk lbracket;
+ },
+ .Identifier => blk: {
+ _ = try appendToken(c, .LBracket, "[");
+ _ = try appendToken(c, .Asterisk, "*");
+ const c_ident = try appendToken(c, .Identifier, "c");
+ _ = try appendToken(c, .RBracket, "]");
+ break :blk c_ident;
+ },
+ .Asterisk => try appendToken(c, .Asterisk, "*"),
+ else => unreachable,
+ };
node.* = ast.Node.PrefixOp{
.base = ast.Node{ .id = .PrefixOp },
- .op_token = try appendToken(c, op_tok_id, bytes),
+ .op_token = op_token,
.op = ast.Node.PrefixOp.Op{
- .PtrType = ast.Node.PrefixOp.PtrInfo{
- .allowzero_token = null,
- .align_info = null,
+ .PtrType = .{
.const_token = if (is_const) try appendToken(c, .Keyword_const, "const") else null,
.volatile_token = if (is_volatile) try appendToken(c, .Keyword_volatile, "volatile") else null,
- .null_token = null,
},
},
.rhs = undefined, // translate and set afterward
@@ -1226,7 +1239,6 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour
ZigClangQualType_isConstQualified(child_qt),
ZigClangQualType_isVolatileQualified(child_qt),
.Asterisk,
- "*",
);
optional_node.rhs = &pointer_node.base;
pointer_node.rhs = try transQualType(rp, child_qt, source_loc);
@@ -1236,8 +1248,7 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour
rp.c,
ZigClangQualType_isConstQualified(child_qt),
ZigClangQualType_isVolatileQualified(child_qt),
- .BracketStarCBracket,
- "[*c]",
+ .Identifier,
);
pointer_node.rhs = try transQualType(rp, child_qt, source_loc);
return &pointer_node.base;