Commit 4dca99d3f6
Changed files (4)
lib
lib/std/zig/ast.zig
@@ -9,48 +9,30 @@ const testing = std.testing;
const mem = std.mem;
const Token = std.zig.Token;
-pub const TokenIndex = usize;
-pub const NodeIndex = usize;
+pub const TokenIndex = u32;
+pub const ByteOffset = u32;
+
+pub const TokenList = std.MultiArrayList(struct {
+ tag: Token.Tag,
+ start: ByteOffset,
+});
+pub const NodeList = std.MultiArrayList(struct {
+ tag: Node.Tag,
+ main_token: TokenIndex,
+ data: Node.Data,
+});
pub const Tree = struct {
/// Reference to externally-owned data.
source: []const u8,
- token_ids: []const Token.Id,
- token_locs: []const Token.Loc,
- errors: []const Error,
- root_node: *Node.Root,
-
- arena: std.heap.ArenaAllocator.State,
- gpa: *mem.Allocator,
-
- /// translate-c uses this to avoid having to emit correct newlines
- /// TODO get rid of this hack
- generated: bool = false,
-
- pub fn deinit(self: *Tree) void {
- self.gpa.free(self.token_ids);
- self.gpa.free(self.token_locs);
- self.gpa.free(self.errors);
- self.arena.promote(self.gpa).deinit();
- }
-
- pub fn renderError(self: *Tree, parse_error: *const Error, stream: anytype) !void {
- return parse_error.render(self.token_ids, stream);
- }
-
- pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {
- return self.tokenSliceLoc(self.token_locs[token_index]);
- }
- pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 {
- return self.source[token.start..token.end];
- }
+ tokens: TokenList.Slice,
+ /// The root AST node is assumed to be index 0. Since there can be no
+ /// references to the root node, this means 0 is available to indicate null.
+ nodes: NodeList.Slice,
+ extra_data: []Node.Index,
- pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 {
- const first_token = self.token_locs[node.firstToken()];
- const last_token = self.token_locs[node.lastToken()];
- return self.source[first_token.start..last_token.end];
- }
+ errors: []const Error,
pub const Location = struct {
line: usize,
@@ -59,21 +41,28 @@ pub const Tree = struct {
line_end: usize,
};
- /// Return the Location of the token relative to the offset specified by `start_index`.
- pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location {
+ pub fn deinit(tree: *Tree, gpa: *mem.Allocator) void {
+ tree.tokens.deinit(gpa);
+ tree.nodes.deinit(gpa);
+ gpa.free(tree.extra_data);
+ gpa.free(tree.errors);
+ tree.* = undefined;
+ }
+
+ pub fn tokenLocation(self: Tree, start_offset: ByteOffset, token_index: TokenIndex) Location {
var loc = Location{
.line = 0,
.column = 0,
- .line_start = start_index,
+ .line_start = start_offset,
.line_end = self.source.len,
};
- if (self.generated)
- return loc;
- const token_start = token.start;
- for (self.source[start_index..]) |c, i| {
- if (i + start_index == token_start) {
- loc.line_end = i + start_index;
- while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') : (loc.line_end += 1) {}
+ const token_start = self.tokens.items(.start)[token_index];
+ for (self.source[start_offset..]) |c, i| {
+ if (i + start_offset == token_start) {
+ loc.line_end = i + start_offset;
+ while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') {
+ loc.line_end += 1;
+ }
return loc;
}
if (c == '\n') {
@@ -87,94 +76,9 @@ pub const Tree = struct {
return loc;
}
- pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {
- return self.tokenLocationLoc(start_index, self.token_locs[token_index]);
- }
-
- pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {
- return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]);
- }
-
- pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool {
- return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;
- }
-
- pub fn dump(self: *Tree) void {
- self.root_node.base.dump(0);
- }
-
- /// Skips over comments
- pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {
- var index = token_index - 1;
- while (self.token_ids[index] == Token.Id.LineComment) {
- index -= 1;
- }
- return index;
- }
-
- /// Skips over comments
- pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {
- var index = token_index + 1;
- while (self.token_ids[index] == Token.Id.LineComment) {
- index += 1;
- }
- return index;
- }
-};
-
-pub const Error = union(enum) {
- InvalidToken: InvalidToken,
- ExpectedContainerMembers: ExpectedContainerMembers,
- ExpectedStringLiteral: ExpectedStringLiteral,
- ExpectedIntegerLiteral: ExpectedIntegerLiteral,
- ExpectedPubItem: ExpectedPubItem,
- ExpectedIdentifier: ExpectedIdentifier,
- ExpectedStatement: ExpectedStatement,
- ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,
- ExpectedVarDecl: ExpectedVarDecl,
- ExpectedFn: ExpectedFn,
- ExpectedReturnType: ExpectedReturnType,
- ExpectedAggregateKw: ExpectedAggregateKw,
- UnattachedDocComment: UnattachedDocComment,
- ExpectedEqOrSemi: ExpectedEqOrSemi,
- ExpectedSemiOrLBrace: ExpectedSemiOrLBrace,
- ExpectedSemiOrElse: ExpectedSemiOrElse,
- ExpectedLabelOrLBrace: ExpectedLabelOrLBrace,
- ExpectedLBrace: ExpectedLBrace,
- ExpectedColonOrRParen: ExpectedColonOrRParen,
- ExpectedLabelable: ExpectedLabelable,
- ExpectedInlinable: ExpectedInlinable,
- ExpectedAsmOutputReturnOrType: ExpectedAsmOutputReturnOrType,
- ExpectedCall: ExpectedCall,
- ExpectedCallOrFnProto: ExpectedCallOrFnProto,
- ExpectedSliceOrRBracket: ExpectedSliceOrRBracket,
- ExtraAlignQualifier: ExtraAlignQualifier,
- ExtraConstQualifier: ExtraConstQualifier,
- ExtraVolatileQualifier: ExtraVolatileQualifier,
- ExtraAllowZeroQualifier: ExtraAllowZeroQualifier,
- ExpectedTypeExpr: ExpectedTypeExpr,
- ExpectedPrimaryTypeExpr: ExpectedPrimaryTypeExpr,
- ExpectedParamType: ExpectedParamType,
- ExpectedExpr: ExpectedExpr,
- ExpectedPrimaryExpr: ExpectedPrimaryExpr,
- ExpectedToken: ExpectedToken,
- ExpectedCommaOrEnd: ExpectedCommaOrEnd,
- ExpectedParamList: ExpectedParamList,
- ExpectedPayload: ExpectedPayload,
- ExpectedBlockOrAssignment: ExpectedBlockOrAssignment,
- ExpectedBlockOrExpression: ExpectedBlockOrExpression,
- ExpectedExprOrAssignment: ExpectedExprOrAssignment,
- ExpectedPrefixExpr: ExpectedPrefixExpr,
- ExpectedLoopExpr: ExpectedLoopExpr,
- ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,
- ExpectedSuffixOp: ExpectedSuffixOp,
- ExpectedBlockOrField: ExpectedBlockOrField,
- DeclBetweenFields: DeclBetweenFields,
- InvalidAnd: InvalidAnd,
- AsteriskAfterPointerDereference: AsteriskAfterPointerDereference,
-
- pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void {
- switch (self.*) {
+ pub fn renderError(tree: Tree, parse_error: Error, stream: anytype) !void {
+ const tokens = tree.tokens.items(.tag);
+ switch (parse_error) {
.InvalidToken => |*x| return x.render(tokens, stream),
.ExpectedContainerMembers => |*x| return x.render(tokens, stream),
.ExpectedStringLiteral => |*x| return x.render(tokens, stream),
@@ -197,8 +101,8 @@ pub const Error = union(enum) {
.ExpectedLabelable => |*x| return x.render(tokens, stream),
.ExpectedInlinable => |*x| return x.render(tokens, stream),
.ExpectedAsmOutputReturnOrType => |*x| return x.render(tokens, stream),
- .ExpectedCall => |*x| return x.render(tokens, stream),
- .ExpectedCallOrFnProto => |*x| return x.render(tokens, stream),
+ .ExpectedCall => |x| return x.render(tree, stream),
+ .ExpectedCallOrFnProto => |x| return x.render(tree, stream),
.ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream),
.ExtraAlignQualifier => |*x| return x.render(tokens, stream),
.ExtraConstQualifier => |*x| return x.render(tokens, stream),
@@ -227,8 +131,8 @@ pub const Error = union(enum) {
}
}
- pub fn loc(self: *const Error) TokenIndex {
- switch (self.*) {
+ pub fn errorToken(tree: Tree, parse_error: Error) TokenIndex {
+ switch (parse_error) {
.InvalidToken => |x| return x.token,
.ExpectedContainerMembers => |x| return x.token,
.ExpectedStringLiteral => |x| return x.token,
@@ -251,8 +155,8 @@ pub const Error = union(enum) {
.ExpectedLabelable => |x| return x.token,
.ExpectedInlinable => |x| return x.token,
.ExpectedAsmOutputReturnOrType => |x| return x.token,
- .ExpectedCall => |x| return x.node.firstToken(),
- .ExpectedCallOrFnProto => |x| return x.node.firstToken(),
+ .ExpectedCall => |x| return tree.nodes.items(.main_token)[x.node],
+ .ExpectedCallOrFnProto => |x| return tree.nodes.items(.main_token)[x.node],
.ExpectedSliceOrRBracket => |x| return x.token,
.ExtraAlignQualifier => |x| return x.token,
.ExtraConstQualifier => |x| return x.token,
@@ -281,6 +185,78 @@ pub const Error = union(enum) {
}
}
+ /// Skips over comments.
+ pub fn prevToken(self: *const Tree, token_index: TokenIndex) TokenIndex {
+ const token_tags = self.tokens.items(.tag);
+ var index = token_index - 1;
+ while (token_tags[index] == .LineComment) {
+ index -= 1;
+ }
+ return index;
+ }
+
+ /// Skips over comments.
+ pub fn nextToken(self: *const Tree, token_index: TokenIndex) TokenIndex {
+ const token_tags = self.tokens.items(.tag);
+ var index = token_index + 1;
+ while (token_tags[index] == .LineComment) {
+ index += 1;
+ }
+ return index;
+ }
+};
+
+pub const Error = union(enum) {
+ InvalidToken: InvalidToken,
+ ExpectedContainerMembers: ExpectedContainerMembers,
+ ExpectedStringLiteral: ExpectedStringLiteral,
+ ExpectedIntegerLiteral: ExpectedIntegerLiteral,
+ ExpectedPubItem: ExpectedPubItem,
+ ExpectedIdentifier: ExpectedIdentifier,
+ ExpectedStatement: ExpectedStatement,
+ ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,
+ ExpectedVarDecl: ExpectedVarDecl,
+ ExpectedFn: ExpectedFn,
+ ExpectedReturnType: ExpectedReturnType,
+ ExpectedAggregateKw: ExpectedAggregateKw,
+ UnattachedDocComment: UnattachedDocComment,
+ ExpectedEqOrSemi: ExpectedEqOrSemi,
+ ExpectedSemiOrLBrace: ExpectedSemiOrLBrace,
+ ExpectedSemiOrElse: ExpectedSemiOrElse,
+ ExpectedLabelOrLBrace: ExpectedLabelOrLBrace,
+ ExpectedLBrace: ExpectedLBrace,
+ ExpectedColonOrRParen: ExpectedColonOrRParen,
+ ExpectedLabelable: ExpectedLabelable,
+ ExpectedInlinable: ExpectedInlinable,
+ ExpectedAsmOutputReturnOrType: ExpectedAsmOutputReturnOrType,
+ ExpectedCall: ExpectedCall,
+ ExpectedCallOrFnProto: ExpectedCallOrFnProto,
+ ExpectedSliceOrRBracket: ExpectedSliceOrRBracket,
+ ExtraAlignQualifier: ExtraAlignQualifier,
+ ExtraConstQualifier: ExtraConstQualifier,
+ ExtraVolatileQualifier: ExtraVolatileQualifier,
+ ExtraAllowZeroQualifier: ExtraAllowZeroQualifier,
+ ExpectedTypeExpr: ExpectedTypeExpr,
+ ExpectedPrimaryTypeExpr: ExpectedPrimaryTypeExpr,
+ ExpectedParamType: ExpectedParamType,
+ ExpectedExpr: ExpectedExpr,
+ ExpectedPrimaryExpr: ExpectedPrimaryExpr,
+ ExpectedToken: ExpectedToken,
+ ExpectedCommaOrEnd: ExpectedCommaOrEnd,
+ ExpectedParamList: ExpectedParamList,
+ ExpectedPayload: ExpectedPayload,
+ ExpectedBlockOrAssignment: ExpectedBlockOrAssignment,
+ ExpectedBlockOrExpression: ExpectedBlockOrExpression,
+ ExpectedExprOrAssignment: ExpectedExprOrAssignment,
+ ExpectedPrefixExpr: ExpectedPrefixExpr,
+ ExpectedLoopExpr: ExpectedLoopExpr,
+ ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,
+ ExpectedSuffixOp: ExpectedSuffixOp,
+ ExpectedBlockOrField: ExpectedBlockOrField,
+ DeclBetweenFields: DeclBetweenFields,
+ InvalidAnd: InvalidAnd,
+ AsteriskAfterPointerDereference: AsteriskAfterPointerDereference,
+
pub const InvalidToken = SingleTokenError("Invalid token '{s}'");
pub const ExpectedContainerMembers = SingleTokenError("Expected test, comptime, var decl, or container field, found '{s}'");
pub const ExpectedStringLiteral = SingleTokenError("Expected string literal, found '{s}'");
@@ -291,7 +267,7 @@ pub const Error = union(enum) {
pub const ExpectedVarDecl = SingleTokenError("Expected variable declaration, found '{s}'");
pub const ExpectedFn = SingleTokenError("Expected function, found '{s}'");
pub const ExpectedReturnType = SingleTokenError("Expected 'var' or return type expression, found '{s}'");
- pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Id.Keyword_struct.symbol() ++ "', '" ++ Token.Id.Keyword_union.symbol() ++ "', '" ++ Token.Id.Keyword_enum.symbol() ++ "', or '" ++ Token.Id.Keyword_opaque.symbol() ++ "', found '{s}'");
+ pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Tag.Keyword_struct.symbol() ++ "', '" ++ Token.Tag.Keyword_union.symbol() ++ "', '" ++ Token.Tag.Keyword_enum.symbol() ++ "', or '" ++ Token.Tag.Keyword_opaque.symbol() ++ "', found '{s}'");
pub const ExpectedEqOrSemi = SingleTokenError("Expected '=' or ';', found '{s}'");
pub const ExpectedSemiOrLBrace = SingleTokenError("Expected ';' or '{{', found '{s}'");
pub const ExpectedSemiOrElse = SingleTokenError("Expected ';' or 'else', found '{s}'");
@@ -300,7 +276,7 @@ pub const Error = union(enum) {
pub const ExpectedColonOrRParen = SingleTokenError("Expected ':' or ')', found '{s}'");
pub const ExpectedLabelable = SingleTokenError("Expected 'while', 'for', 'inline', 'suspend', or '{{', found '{s}'");
pub const ExpectedInlinable = SingleTokenError("Expected 'while' or 'for', found '{s}'");
- pub const ExpectedAsmOutputReturnOrType = SingleTokenError("Expected '->' or '" ++ Token.Id.Identifier.symbol() ++ "', found '{s}'");
+ pub const ExpectedAsmOutputReturnOrType = SingleTokenError("Expected '->' or '" ++ Token.Tag.Identifier.symbol() ++ "', found '{s}'");
pub const ExpectedSliceOrRBracket = SingleTokenError("Expected ']' or '..', found '{s}'");
pub const ExpectedTypeExpr = SingleTokenError("Expected type expression, found '{s}'");
pub const ExpectedPrimaryTypeExpr = SingleTokenError("Expected primary type expression, found '{s}'");
@@ -329,29 +305,31 @@ pub const Error = union(enum) {
pub const AsteriskAfterPointerDereference = SimpleError("`.*` can't be followed by `*`. Are you missing a space?");
pub const ExpectedCall = struct {
- node: *Node,
+ node: Node.Index,
- pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: ExpectedCall, tree: Tree, stream: anytype) !void {
+ const node_tag = tree.nodes.items(.tag)[self.node];
return stream.print("expected " ++ @tagName(Node.Tag.Call) ++ ", found {s}", .{
- @tagName(self.node.tag),
+ @tagName(node_tag),
});
}
};
pub const ExpectedCallOrFnProto = struct {
- node: *Node,
+ node: Node.Index,
- pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: ExpectedCallOrFnProto, tree: Tree, stream: anytype) !void {
+ const node_tag = tree.nodes.items(.tag)[self.node];
return stream.print("expected " ++ @tagName(Node.Tag.Call) ++ " or " ++
- @tagName(Node.Tag.FnProto) ++ ", found {s}", .{@tagName(self.node.tag)});
+ @tagName(Node.Tag.FnProto) ++ ", found {s}", .{@tagName(node_tag)});
}
};
pub const ExpectedToken = struct {
token: TokenIndex,
- expected_id: Token.Id,
+ expected_id: Token.Tag,
- pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: *const ExpectedToken, tokens: []const Token.Tag, stream: anytype) !void {
const found_token = tokens[self.token];
switch (found_token) {
.Invalid => {
@@ -367,9 +345,9 @@ pub const Error = union(enum) {
pub const ExpectedCommaOrEnd = struct {
token: TokenIndex,
- end_id: Token.Id,
+ end_id: Token.Tag,
- pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Tag, stream: anytype) !void {
const actual_token = tokens[self.token];
return stream.print("expected ',' or '{s}', found '{s}'", .{
self.end_id.symbol(),
@@ -384,7 +362,7 @@ pub const Error = union(enum) {
token: TokenIndex,
- pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: *const ThisError, tokens: []const Token.Tag, stream: anytype) !void {
const actual_token = tokens[self.token];
return stream.print(msg, .{actual_token.symbol()});
}
@@ -397,2886 +375,466 @@ pub const Error = union(enum) {
token: TokenIndex,
- pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: anytype) !void {
+ pub fn render(self: *const ThisError, tokens: []const Token.Tag, stream: anytype) !void {
return stream.writeAll(msg);
}
};
}
+
+ pub fn loc(self: Error) TokenIndex {
+ switch (self) {
+ .InvalidToken => |x| return x.token,
+ .ExpectedContainerMembers => |x| return x.token,
+ .ExpectedStringLiteral => |x| return x.token,
+ .ExpectedIntegerLiteral => |x| return x.token,
+ .ExpectedPubItem => |x| return x.token,
+ .ExpectedIdentifier => |x| return x.token,
+ .ExpectedStatement => |x| return x.token,
+ .ExpectedVarDeclOrFn => |x| return x.token,
+ .ExpectedVarDecl => |x| return x.token,
+ .ExpectedFn => |x| return x.token,
+ .ExpectedReturnType => |x| return x.token,
+ .ExpectedAggregateKw => |x| return x.token,
+ .UnattachedDocComment => |x| return x.token,
+ .ExpectedEqOrSemi => |x| return x.token,
+ .ExpectedSemiOrLBrace => |x| return x.token,
+ .ExpectedSemiOrElse => |x| return x.token,
+ .ExpectedLabelOrLBrace => |x| return x.token,
+ .ExpectedLBrace => |x| return x.token,
+ .ExpectedColonOrRParen => |x| return x.token,
+ .ExpectedLabelable => |x| return x.token,
+ .ExpectedInlinable => |x| return x.token,
+ .ExpectedAsmOutputReturnOrType => |x| return x.token,
+ .ExpectedCall => |x| @panic("TODO redo ast errors"),
+ .ExpectedCallOrFnProto => |x| @panic("TODO redo ast errors"),
+ .ExpectedSliceOrRBracket => |x| return x.token,
+ .ExtraAlignQualifier => |x| return x.token,
+ .ExtraConstQualifier => |x| return x.token,
+ .ExtraVolatileQualifier => |x| return x.token,
+ .ExtraAllowZeroQualifier => |x| return x.token,
+ .ExpectedTypeExpr => |x| return x.token,
+ .ExpectedPrimaryTypeExpr => |x| return x.token,
+ .ExpectedParamType => |x| return x.token,
+ .ExpectedExpr => |x| return x.token,
+ .ExpectedPrimaryExpr => |x| return x.token,
+ .ExpectedToken => |x| return x.token,
+ .ExpectedCommaOrEnd => |x| return x.token,
+ .ExpectedParamList => |x| return x.token,
+ .ExpectedPayload => |x| return x.token,
+ .ExpectedBlockOrAssignment => |x| return x.token,
+ .ExpectedBlockOrExpression => |x| return x.token,
+ .ExpectedExprOrAssignment => |x| return x.token,
+ .ExpectedPrefixExpr => |x| return x.token,
+ .ExpectedLoopExpr => |x| return x.token,
+ .ExpectedDerefOrUnwrap => |x| return x.token,
+ .ExpectedSuffixOp => |x| return x.token,
+ .ExpectedBlockOrField => |x| return x.token,
+ .DeclBetweenFields => |x| return x.token,
+ .InvalidAnd => |x| return x.token,
+ .AsteriskAfterPointerDereference => |x| return x.token,
+ }
+ }
};
pub const Node = struct {
- tag: Tag,
+ index: Index,
+
+ pub const Index = u32;
+
+ comptime {
+ // Goal is to keep this under one byte for efficiency.
+ assert(@sizeOf(Tag) == 1);
+ }
pub const Tag = enum {
- // Top level
+ /// sub_list[lhs...rhs]
Root,
- Use,
+ /// lhs is the sub-expression. rhs is unused.
+ UsingNamespace,
+ /// lhs is test name token (must be string literal), if any.
+ /// rhs is the body node.
TestDecl,
-
- // Statements
- VarDecl,
+ /// lhs is the index into global_var_decl_list.
+ /// rhs is the initialization expression, if any.
+ GlobalVarDecl,
+ /// `var a: x align(y) = rhs`
+ /// lhs is the index into local_var_decl_list.
+ LocalVarDecl,
+ /// `var a: lhs = rhs`. lhs and rhs may be unused.
+ /// Can be local or global.
+ SimpleVarDecl,
+ /// `var a align(lhs) = rhs`. lhs and rhs may be unused.
+ /// Can be local or global.
+ AlignedVarDecl,
+ /// lhs is the identifier token payload if any,
+ /// rhs is the deferred expression.
+ ErrDefer,
+ /// lhs is unused.
+ /// rhs is the deferred expression.
Defer,
-
- // Infix operators
+ /// lhs is target expr; rhs is fallback expr.
+ /// payload is determined by looking at the prev tokens before rhs.
Catch,
-
- // SimpleInfixOp
- Add,
- AddWrap,
- ArrayCat,
- ArrayMult,
- Assign,
- AssignBitAnd,
- AssignBitOr,
- AssignBitShiftLeft,
- AssignBitShiftRight,
- AssignBitXor,
+ /// `lhs.a`. main_token is the dot. rhs is the identifier token index.
+ FieldAccess,
+ /// `lhs.?`. main_token is the dot. rhs is the `?` token index.
+ UnwrapOptional,
+ /// `lhs == rhs`. main_token is op.
+ EqualEqual,
+ /// `lhs != rhs`. main_token is op.
+ BangEqual,
+ /// `lhs < rhs`. main_token is op.
+ LessThan,
+ /// `lhs > rhs`. main_token is op.
+ GreaterThan,
+ /// `lhs <= rhs`. main_token is op.
+ LessOrEqual,
+ /// `lhs >= rhs`. main_token is op.
+ GreaterOrEqual,
+ /// `lhs *= rhs`. main_token is op.
+ AssignMul,
+ /// `lhs /= rhs`. main_token is op.
AssignDiv,
- AssignSub,
- AssignSubWrap,
+ /// `lhs *= rhs`. main_token is op.
AssignMod,
+ /// `lhs += rhs`. main_token is op.
AssignAdd,
- AssignAddWrap,
- AssignMul,
+ /// `lhs -= rhs`. main_token is op.
+ AssignSub,
+ /// `lhs <<= rhs`. main_token is op.
+ AssignBitShiftLeft,
+ /// `lhs >>= rhs`. main_token is op.
+ AssignBitShiftRight,
+ /// `lhs &= rhs`. main_token is op.
+ AssignBitAnd,
+ /// `lhs ^= rhs`. main_token is op.
+ AssignBitXor,
+ /// `lhs |= rhs`. main_token is op.
+ AssignBitOr,
+ /// `lhs *%= rhs`. main_token is op.
AssignMulWrap,
- BangEqual,
- BitAnd,
- BitOr,
+ /// `lhs +%= rhs`. main_token is op.
+ AssignAddWrap,
+ /// `lhs -%= rhs`. main_token is op.
+ AssignSubWrap,
+ /// `lhs = rhs`. main_token is op.
+ Assign,
+ /// `lhs || rhs`. main_token is the `||`.
+ MergeErrorSets,
+ /// `lhs * rhs`. main_token is the `*`.
+ Mul,
+ /// `lhs / rhs`. main_token is the `/`.
+ Div,
+ /// `lhs % rhs`. main_token is the `%`.
+ Mod,
+ /// `lhs ** rhs`. main_token is the `**`.
+ ArrayMult,
+ /// `lhs *% rhs`. main_token is the `*%`.
+ MulWrap,
+ /// `lhs + rhs`. main_token is the `+`.
+ Add,
+ /// `lhs - rhs`. main_token is the `-`.
+ Sub,
+ /// `lhs ++ rhs`. main_token is the `++`.
+ ArrayCat,
+ /// `lhs +% rhs`. main_token is the `+%`.
+ AddWrap,
+ /// `lhs -% rhs`. main_token is the `-%`.
+ SubWrap,
+ /// `lhs << rhs`. main_token is the `<<`.
BitShiftLeft,
+ /// `lhs >> rhs`. main_token is the `>>`.
BitShiftRight,
+ /// `lhs & rhs`. main_token is the `&`.
+ BitAnd,
+ /// `lhs ^ rhs`. main_token is the `^`.
BitXor,
+ /// `lhs | rhs`. main_token is the `|`.
+ BitOr,
+ /// `lhs orelse rhs`. main_token is the `orelse`.
+ OrElse,
+ /// `lhs and rhs`. main_token is the `and`.
BoolAnd,
+ /// `lhs or rhs`. main_token is the `or`.
BoolOr,
- Div,
- EqualEqual,
- ErrorUnion,
- GreaterOrEqual,
- GreaterThan,
- LessOrEqual,
- LessThan,
- MergeErrorSets,
- Mod,
- Mul,
- MulWrap,
- Period,
- Range,
- Sub,
- SubWrap,
- OrElse,
-
- // SimplePrefixOp
- AddressOf,
- Await,
- BitNot,
+ /// `op lhs`. rhs unused. main_token is op.
BoolNot,
- OptionalType,
+ /// `op lhs`. rhs unused. main_token is op.
Negation,
+ /// `op lhs`. rhs unused. main_token is op.
+ BitNot,
+ /// `op lhs`. rhs unused. main_token is op.
NegationWrap,
- Resume,
+ /// `op lhs`. rhs unused. main_token is op.
+ AddressOf,
+ /// `op lhs`. rhs unused. main_token is op.
Try,
-
+ /// `op lhs`. rhs unused. main_token is op.
+ Await,
+ /// `?lhs`. rhs unused. main_token is the `?`.
+ OptionalType,
+ /// `[lhs]rhs`. lhs can be omitted to make it a slice.
ArrayType,
- /// ArrayType but has a sentinel node.
+ /// `[lhs:a]b`. `ArrayTypeSentinel[rhs]`.
ArrayTypeSentinel,
+ /// `[*]align(lhs) rhs`. lhs can be omitted.
+ /// `*align(lhs) rhs`. lhs can be omitted.
+ /// `[]rhs`.
+ PtrTypeAligned,
+ /// `[*:lhs]rhs`. lhs can be omitted.
+ /// `*rhs`.
+ /// `[:lhs]rhs`.
+ PtrTypeSentinel,
+ /// lhs is index into PtrType. rhs is the element type expression.
PtrType,
+ /// lhs is index into SliceType. rhs is the element type expression.
+ /// Can be pointer or slice, depending on main_token.
SliceType,
- /// `a[b..c]`
+ /// `lhs[rhs..]`
+ /// main_token is the `[`.
+ SliceOpen,
+ /// `lhs[b..c :d]`. `slice_list[rhs]`.
+ /// main_token is the `[`.
Slice,
- /// `a.*`
+ /// `lhs.*`. rhs is unused.
Deref,
- /// `a.?`
- UnwrapOptional,
- /// `a[b]`
+ /// `lhs[rhs]`.
ArrayAccess,
- /// `T{a, b}`
- ArrayInitializer,
- /// ArrayInitializer but with `.` instead of a left-hand-side operand.
- ArrayInitializerDot,
- /// `T{.a = b}`
- StructInitializer,
- /// StructInitializer but with `.` instead of a left-hand-side operand.
- StructInitializerDot,
- /// `foo()`
+ /// `lhs{rhs}`. rhs can be omitted.
+ ArrayInitOne,
+ /// `.{lhs, rhs}`. lhs and rhs can be omitted.
+ ArrayInitDotTwo,
+ /// `.{a, b}`. `sub_list[lhs..rhs]`.
+ ArrayInitDot,
+ /// `lhs{a, b}`. `sub_range_list[rhs]`. lhs can be omitted which means `.{a, b}`.
+ ArrayInit,
+ /// `lhs{.a = rhs}`. rhs can be omitted making it empty.
+ StructInitOne,
+ /// `.{.a = lhs, .b = rhs}`. lhs and rhs can be omitted.
+ StructInitDotTwo,
+ /// `.{.a = b, .c = d}`. `sub_list[lhs..rhs]`.
+ StructInitDot,
+ /// `lhs{.a = b, .c = d}`. `sub_range_list[rhs]`.
+ /// lhs can be omitted which means `.{.a = b, .c = d}`.
+ StructInit,
+ /// `lhs(rhs)`. rhs can be omitted.
+ CallOne,
+ /// `lhs(a, b, c)`. `sub_range_list[rhs]`.
+ /// main_token is the `(`.
Call,
-
- // Control flow
+ /// `switch(lhs) {}`. `sub_range_list[rhs]`.
Switch,
+ /// `lhs => rhs`. If lhs is omitted it means `else`.
+ /// main_token is the `=>`
+ SwitchCaseOne,
+ /// `a, b, c => rhs`. `sub_range_list[lhs]`.
+ SwitchCaseMulti,
+ /// `lhs...rhs`.
+ SwitchRange,
+ /// `while (lhs) rhs`.
+ WhileSimple,
+ /// `while (lhs) |x| rhs`.
+ WhileSimpleOptional,
+ /// `while (lhs) : (a) b`. `WhileCont[rhs]`.
+ WhileCont,
+ /// `while (lhs) : (a) b`. `WhileCont[rhs]`.
+ WhileContOptional,
+ /// `while (lhs) : (a) b else c`. `While[rhs]`.
While,
+ /// `while (lhs) |x| : (a) b else c`. `While[rhs]`.
+ WhileOptional,
+ /// `while (lhs) |x| : (a) b else |y| c`. `While[rhs]`.
+ WhileError,
+ /// `for (lhs) rhs`.
+ ForSimple,
+ /// `for (lhs) a else b`. `if_list[rhs]`.
For,
+ /// `if (lhs) rhs`.
+ IfSimple,
+ /// `if (lhs) |a| rhs`.
+ IfSimpleOptional,
+ /// `if (lhs) a else b`. `if_list[rhs]`.
If,
+ /// `if (lhs) |x| a else b`. `if_list[rhs]`.
+ IfOptional,
+ /// `if (lhs) |x| a else |y| b`. `if_list[rhs]`.
+ IfError,
+ /// `suspend lhs`. lhs can be omitted. rhs is unused.
Suspend,
+ /// `resume lhs`. rhs is unused.
+ Resume,
+ /// `continue`. lhs is token index of label if any. rhs is unused.
Continue,
+ /// `break rhs`. rhs can be omitted. lhs is label token index, if any.
Break,
+ /// `return lhs`. lhs can be omitted. rhs is unused.
Return,
-
- // Type expressions
- AnyType,
- ErrorType,
+ /// `fn(a: lhs) rhs`. lhs can be omitted.
+ /// anytype and ... parameters are omitted from the AST tree.
+ FnProtoSimple,
+ /// `fn(a: b, c: d) rhs`. `sub_range_list[lhs]`.
+ /// anytype and ... parameters are omitted from the AST tree.
+ FnProtoSimpleMulti,
+ /// `fn(a: b) rhs linksection(e) callconv(f)`. lhs is index into extra_data.
+ /// zero or one parameters.
+ /// anytype and ... parameters are omitted from the AST tree.
+ FnProtoOne,
+ /// `fn(a: b, c: d) rhs linksection(e) callconv(f)`. `fn_proto_list[lhs]`.
+ /// anytype and ... parameters are omitted from the AST tree.
FnProto,
+ /// lhs is the FnProto, rhs is the function body block.
+ FnDecl,
+ /// `anyframe->rhs`. main_token is `anyframe`. `lhs` is arrow token index.
AnyFrameType,
-
- // Primary expressions
- IntegerLiteral,
- FloatLiteral,
+ /// Could be integer literal, float literal, char literal, bool literal,
+ /// null literal, undefined literal, unreachable, depending on the token.
+ /// Both lhs and rhs unused.
+ OneToken,
+ /// Both lhs and rhs unused.
+ /// Most identifiers will not have explicit AST nodes, however for expressions
+ /// which could be one of many different kinds of AST nodes, there will be an
+ /// Identifier AST node for it.
+ Identifier,
+ /// lhs is the dot token index, rhs unused, main_token is the identifier.
EnumLiteral,
- StringLiteral,
+ /// Both lhs and rhs unused.
MultilineStringLiteral,
- CharLiteral,
- BoolLiteral,
- NullLiteral,
- UndefinedLiteral,
- Unreachable,
- Identifier,
+ /// `(lhs)`. main_token is the `(`; rhs is the token index of the `)`.
GroupedExpression,
+ /// `@a(lhs, rhs)`. lhs and rhs may be omitted.
+ BuiltinCallTwo,
+ /// `@a(b, c)`. `sub_list[lhs..rhs]`.
BuiltinCall,
+ /// `error{a, b}`.
+ /// lhs and rhs both unused.
ErrorSetDecl,
+ /// `struct {}`, `union {}`, etc. `sub_list[lhs..rhs]`.
ContainerDecl,
- Asm,
+ /// `union(lhs)` / `enum(lhs)`. `sub_range_list[rhs]`.
+ ContainerDeclArg,
+ /// `union(enum) {}`. `sub_list[lhs..rhs]`.
+ /// Note that tagged unions with explicitly provided enums are represented
+ /// by `ContainerDeclArg`.
+ TaggedUnion,
+ /// `union(enum(lhs)) {}`. `sub_list_range[rhs]`.
+ TaggedUnionEnumTag,
+ /// `a: lhs = rhs,`. lhs and rhs can be omitted.
+ ContainerFieldInit,
+ /// `a: lhs align(rhs),`. rhs can be omitted.
+ ContainerFieldAlign,
+ /// `a: lhs align(c) = d,`. `container_field_list[rhs]`.
+ ContainerField,
+ /// `anytype`. both lhs and rhs unused.
+ /// Used by `ContainerField`.
+ AnyType,
+ /// `comptime lhs`. rhs unused.
Comptime,
+ /// `nosuspend lhs`. rhs unused.
Nosuspend,
+ /// `{}`. `sub_list[lhs..rhs]`.
Block,
- LabeledBlock,
-
- // Misc
- DocComment,
- SwitchCase, // TODO make this not a child of AST Node
- SwitchElse, // TODO make this not a child of AST Node
- Else, // TODO make this not a child of AST Node
- Payload, // TODO make this not a child of AST Node
- PointerPayload, // TODO make this not a child of AST Node
- PointerIndexPayload, // TODO make this not a child of AST Node
- ContainerField,
- ErrorTag, // TODO make this not a child of AST Node
- FieldInitializer, // TODO make this not a child of AST Node
-
- pub fn Type(tag: Tag) type {
- return switch (tag) {
- .Root => Root,
- .Use => Use,
- .TestDecl => TestDecl,
- .VarDecl => VarDecl,
- .Defer => Defer,
- .Catch => Catch,
-
- .Add,
- .AddWrap,
- .ArrayCat,
- .ArrayMult,
- .Assign,
- .AssignBitAnd,
- .AssignBitOr,
- .AssignBitShiftLeft,
- .AssignBitShiftRight,
- .AssignBitXor,
- .AssignDiv,
- .AssignSub,
- .AssignSubWrap,
- .AssignMod,
- .AssignAdd,
- .AssignAddWrap,
- .AssignMul,
- .AssignMulWrap,
- .BangEqual,
- .BitAnd,
- .BitOr,
- .BitShiftLeft,
- .BitShiftRight,
- .BitXor,
- .BoolAnd,
- .BoolOr,
- .Div,
- .EqualEqual,
- .ErrorUnion,
- .GreaterOrEqual,
- .GreaterThan,
- .LessOrEqual,
- .LessThan,
- .MergeErrorSets,
- .Mod,
- .Mul,
- .MulWrap,
- .Period,
- .Range,
- .Sub,
- .SubWrap,
- .OrElse,
- => SimpleInfixOp,
-
- .AddressOf,
- .Await,
- .BitNot,
- .BoolNot,
- .OptionalType,
- .Negation,
- .NegationWrap,
- .Resume,
- .Try,
- => SimplePrefixOp,
-
- .Identifier,
- .BoolLiteral,
- .NullLiteral,
- .UndefinedLiteral,
- .Unreachable,
- .AnyType,
- .ErrorType,
- .IntegerLiteral,
- .FloatLiteral,
- .StringLiteral,
- .CharLiteral,
- => OneToken,
-
- .Continue,
- .Break,
- .Return,
- => ControlFlowExpression,
-
- .ArrayType => ArrayType,
- .ArrayTypeSentinel => ArrayTypeSentinel,
-
- .PtrType => PtrType,
- .SliceType => SliceType,
- .Slice => Slice,
- .Deref, .UnwrapOptional => SimpleSuffixOp,
- .ArrayAccess => ArrayAccess,
-
- .ArrayInitializer => ArrayInitializer,
- .ArrayInitializerDot => ArrayInitializerDot,
-
- .StructInitializer => StructInitializer,
- .StructInitializerDot => StructInitializerDot,
-
- .Call => Call,
- .Switch => Switch,
- .While => While,
- .For => For,
- .If => If,
- .Suspend => Suspend,
- .FnProto => FnProto,
- .AnyFrameType => AnyFrameType,
- .EnumLiteral => EnumLiteral,
- .MultilineStringLiteral => MultilineStringLiteral,
- .GroupedExpression => GroupedExpression,
- .BuiltinCall => BuiltinCall,
- .ErrorSetDecl => ErrorSetDecl,
- .ContainerDecl => ContainerDecl,
- .Asm => Asm,
- .Comptime => Comptime,
- .Nosuspend => Nosuspend,
- .Block => Block,
- .LabeledBlock => LabeledBlock,
- .DocComment => DocComment,
- .SwitchCase => SwitchCase,
- .SwitchElse => SwitchElse,
- .Else => Else,
- .Payload => Payload,
- .PointerPayload => PointerPayload,
- .PointerIndexPayload => PointerIndexPayload,
- .ContainerField => ContainerField,
- .ErrorTag => ErrorTag,
- .FieldInitializer => FieldInitializer,
- };
- }
-
- pub fn isBlock(tag: Tag) bool {
- return switch (tag) {
- .Block, .LabeledBlock => true,
- else => false,
- };
- }
+ /// `asm(lhs)`. rhs unused.
+ AsmSimple,
+ /// `asm(lhs, a)`. `sub_range_list[rhs]`.
+ Asm,
+ /// `[a] "b" (c)`. lhs is string literal token index, rhs is 0.
+ /// `[a] "b" (-> rhs)`. lhs is the string literal token index, rhs is type expr.
+ /// main_token is `a`.
+ AsmOutput,
+ /// `[a] "b" (rhs)`. lhs is string literal token index.
+ /// main_token is `a`.
+ AsmInput,
+ /// `error.a`. lhs is token index of `.`. rhs is token index of `a`.
+ ErrorValue,
+ /// `lhs!rhs`. main_token is the `!`.
+ ErrorUnion,
};
- /// Prefer `castTag` to this.
- pub fn cast(base: *Node, comptime T: type) ?*T {
- if (std.meta.fieldInfo(T, .base).default_value) |default_base| {
- return base.castTag(default_base.tag);
- }
- inline for (@typeInfo(Tag).Enum.fields) |field| {
- const tag = @intToEnum(Tag, field.value);
- if (base.tag == tag) {
- if (T == tag.Type()) {
- return @fieldParentPtr(T, "base", base);
- }
- return null;
- }
- }
- unreachable;
- }
-
- pub fn castTag(base: *Node, comptime tag: Tag) ?*tag.Type() {
- if (base.tag == tag) {
- return @fieldParentPtr(tag.Type(), "base", base);
- }
- return null;
- }
-
- pub fn iterate(base: *Node, index: usize) ?*Node {
- inline for (@typeInfo(Tag).Enum.fields) |field| {
- const tag = @intToEnum(Tag, field.value);
- if (base.tag == tag) {
- return @fieldParentPtr(tag.Type(), "base", base).iterate(index);
- }
- }
- unreachable;
- }
-
- pub fn firstToken(base: *const Node) TokenIndex {
- inline for (@typeInfo(Tag).Enum.fields) |field| {
- const tag = @intToEnum(Tag, field.value);
- if (base.tag == tag) {
- return @fieldParentPtr(tag.Type(), "base", base).firstToken();
- }
- }
- unreachable;
- }
-
- pub fn lastToken(base: *const Node) TokenIndex {
- inline for (@typeInfo(Tag).Enum.fields) |field| {
- const tag = @intToEnum(Tag, field.value);
- if (base.tag == tag) {
- return @fieldParentPtr(tag.Type(), "base", base).lastToken();
- }
- }
- unreachable;
- }
-
- pub fn requireSemiColon(base: *const Node) bool {
- var n = base;
- while (true) {
- switch (n.tag) {
- .Root,
- .ContainerField,
- .Block,
- .LabeledBlock,
- .Payload,
- .PointerPayload,
- .PointerIndexPayload,
- .Switch,
- .SwitchCase,
- .SwitchElse,
- .FieldInitializer,
- .DocComment,
- .TestDecl,
- => return false,
-
- .While => {
- const while_node = @fieldParentPtr(While, "base", n);
- if (while_node.@"else") |@"else"| {
- n = &@"else".base;
- continue;
- }
-
- return !while_node.body.tag.isBlock();
- },
- .For => {
- const for_node = @fieldParentPtr(For, "base", n);
- if (for_node.@"else") |@"else"| {
- n = &@"else".base;
- continue;
- }
-
- return !for_node.body.tag.isBlock();
- },
- .If => {
- const if_node = @fieldParentPtr(If, "base", n);
- if (if_node.@"else") |@"else"| {
- n = &@"else".base;
- continue;
- }
-
- return !if_node.body.tag.isBlock();
- },
- .Else => {
- const else_node = @fieldParentPtr(Else, "base", n);
- n = else_node.body;
- continue;
- },
- .Defer => {
- const defer_node = @fieldParentPtr(Defer, "base", n);
- return !defer_node.expr.tag.isBlock();
- },
- .Comptime => {
- const comptime_node = @fieldParentPtr(Comptime, "base", n);
- return !comptime_node.expr.tag.isBlock();
- },
- .Suspend => {
- const suspend_node = @fieldParentPtr(Suspend, "base", n);
- if (suspend_node.body) |body| {
- return !body.tag.isBlock();
- }
-
- return true;
- },
- .Nosuspend => {
- const nosuspend_node = @fieldParentPtr(Nosuspend, "base", n);
- return !nosuspend_node.expr.tag.isBlock();
- },
- else => return true,
- }
- }
- }
-
- /// Asserts the node is a Block or LabeledBlock and returns the statements slice.
- pub fn blockStatements(base: *Node) []*Node {
- if (base.castTag(.Block)) |block| {
- return block.statements();
- } else if (base.castTag(.LabeledBlock)) |labeled_block| {
- return labeled_block.statements();
- } else {
- unreachable;
- }
- }
-
- pub fn findFirstWithId(self: *Node, id: Id) ?*Node {
- if (self.id == id) return self;
- var child_i: usize = 0;
- while (self.iterate(child_i)) |child| : (child_i += 1) {
- if (child.findFirstWithId(id)) |result| return result;
- }
- return null;
- }
-
- pub fn dump(self: *Node, indent: usize) void {
- {
- var i: usize = 0;
- while (i < indent) : (i += 1) {
- std.debug.warn(" ", .{});
- }
- }
- std.debug.warn("{s}\n", .{@tagName(self.tag)});
-
- var child_i: usize = 0;
- while (self.iterate(child_i)) |child| : (child_i += 1) {
- child.dump(indent + 2);
- }
- }
-
- /// The decls data follows this struct in memory as an array of Node pointers.
- pub const Root = struct {
- base: Node = Node{ .tag = .Root },
- eof_token: TokenIndex,
- decls_len: NodeIndex,
-
- /// After this the caller must initialize the decls list.
- pub fn create(allocator: *mem.Allocator, decls_len: NodeIndex, eof_token: TokenIndex) !*Root {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(Root), sizeInBytes(decls_len));
- const self = @ptrCast(*Root, bytes.ptr);
- self.* = .{
- .eof_token = eof_token,
- .decls_len = decls_len,
- };
- return self;
- }
-
- pub fn destroy(self: *Decl, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.decls_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const Root, index: usize) ?*Node {
- var i = index;
-
- if (i < self.decls_len) return self.declsConst()[i];
- return null;
- }
-
- pub fn decls(self: *Root) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(Root);
- return @ptrCast([*]*Node, decls_start)[0..self.decls_len];
- }
-
- pub fn declsConst(self: *const Root) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Root);
- return @ptrCast([*]const *Node, decls_start)[0..self.decls_len];
- }
-
- pub fn firstToken(self: *const Root) TokenIndex {
- if (self.decls_len == 0) return self.eof_token;
- return self.declsConst()[0].firstToken();
- }
-
- pub fn lastToken(self: *const Root) TokenIndex {
- if (self.decls_len == 0) return self.eof_token;
- return self.declsConst()[self.decls_len - 1].lastToken();
- }
-
- fn sizeInBytes(decls_len: NodeIndex) usize {
- return @sizeOf(Root) + @sizeOf(*Node) * @as(usize, decls_len);
- }
+ pub const Data = struct {
+ lhs: Index,
+ rhs: Index,
};
- /// Trailed in memory by possibly many things, with each optional thing
- /// determined by a bit in `trailer_flags`.
- pub const VarDecl = struct {
- base: Node = Node{ .tag = .VarDecl },
- trailer_flags: TrailerFlags,
- mut_token: TokenIndex,
- name_token: TokenIndex,
- semicolon_token: TokenIndex,
-
- pub const TrailerFlags = std.meta.TrailerFlags(struct {
- doc_comments: *DocComment,
- visib_token: TokenIndex,
- thread_local_token: TokenIndex,
- eq_token: TokenIndex,
- comptime_token: TokenIndex,
- extern_export_token: TokenIndex,
- lib_name: *Node,
- type_node: *Node,
- align_node: *Node,
- section_node: *Node,
- init_node: *Node,
- });
-
- pub fn getDocComments(self: *const VarDecl) ?*DocComment {
- return self.getTrailer(.doc_comments);
- }
-
- pub fn setDocComments(self: *VarDecl, value: *DocComment) void {
- self.setTrailer(.doc_comments, value);
- }
-
- pub fn getVisibToken(self: *const VarDecl) ?TokenIndex {
- return self.getTrailer(.visib_token);
- }
-
- pub fn setVisibToken(self: *VarDecl, value: TokenIndex) void {
- self.setTrailer(.visib_token, value);
- }
-
- pub fn getThreadLocalToken(self: *const VarDecl) ?TokenIndex {
- return self.getTrailer(.thread_local_token);
- }
-
- pub fn setThreadLocalToken(self: *VarDecl, value: TokenIndex) void {
- self.setTrailer(.thread_local_token, value);
- }
-
- pub fn getEqToken(self: *const VarDecl) ?TokenIndex {
- return self.getTrailer(.eq_token);
- }
-
- pub fn setEqToken(self: *VarDecl, value: TokenIndex) void {
- self.setTrailer(.eq_token, value);
- }
-
- pub fn getComptimeToken(self: *const VarDecl) ?TokenIndex {
- return self.getTrailer(.comptime_token);
- }
-
- pub fn setComptimeToken(self: *VarDecl, value: TokenIndex) void {
- self.setTrailer(.comptime_token, value);
- }
-
- pub fn getExternExportToken(self: *const VarDecl) ?TokenIndex {
- return self.getTrailer(.extern_export_token);
- }
-
- pub fn setExternExportToken(self: *VarDecl, value: TokenIndex) void {
- self.setTrailer(.extern_export_token, value);
- }
-
- pub fn getLibName(self: *const VarDecl) ?*Node {
- return self.getTrailer(.lib_name);
- }
-
- pub fn setLibName(self: *VarDecl, value: *Node) void {
- self.setTrailer(.lib_name, value);
- }
-
- pub fn getTypeNode(self: *const VarDecl) ?*Node {
- return self.getTrailer(.type_node);
- }
-
- pub fn setTypeNode(self: *VarDecl, value: *Node) void {
- self.setTrailer(.type_node, value);
- }
-
- pub fn getAlignNode(self: *const VarDecl) ?*Node {
- return self.getTrailer(.align_node);
- }
-
- pub fn setAlignNode(self: *VarDecl, value: *Node) void {
- self.setTrailer(.align_node, value);
- }
-
- pub fn getSectionNode(self: *const VarDecl) ?*Node {
- return self.getTrailer(.section_node);
- }
-
- pub fn setSectionNode(self: *VarDecl, value: *Node) void {
- self.setTrailer(.section_node, value);
- }
-
- pub fn getInitNode(self: *const VarDecl) ?*Node {
- return self.getTrailer(.init_node);
- }
-
- pub fn setInitNode(self: *VarDecl, value: *Node) void {
- self.setTrailer(.init_node, value);
- }
-
- pub const RequiredFields = struct {
- mut_token: TokenIndex,
- name_token: TokenIndex,
- semicolon_token: TokenIndex,
- };
-
- fn getTrailer(self: *const VarDecl, comptime field: TrailerFlags.FieldEnum) ?TrailerFlags.Field(field) {
- const trailers_start = @ptrCast([*]const u8, self) + @sizeOf(VarDecl);
- return self.trailer_flags.get(trailers_start, field);
- }
-
- fn setTrailer(self: *VarDecl, comptime field: TrailerFlags.FieldEnum, value: TrailerFlags.Field(field)) void {
- const trailers_start = @ptrCast([*]u8, self) + @sizeOf(VarDecl);
- self.trailer_flags.set(trailers_start, field, value);
- }
-
- pub fn create(allocator: *mem.Allocator, required: RequiredFields, trailers: TrailerFlags.InitStruct) !*VarDecl {
- const trailer_flags = TrailerFlags.init(trailers);
- const bytes = try allocator.alignedAlloc(u8, @alignOf(VarDecl), sizeInBytes(trailer_flags));
- const var_decl = @ptrCast(*VarDecl, bytes.ptr);
- var_decl.* = .{
- .trailer_flags = trailer_flags,
- .mut_token = required.mut_token,
- .name_token = required.name_token,
- .semicolon_token = required.semicolon_token,
- };
- const trailers_start = bytes.ptr + @sizeOf(VarDecl);
- trailer_flags.setMany(trailers_start, trailers);
- return var_decl;
- }
-
- pub fn destroy(self: *VarDecl, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.trailer_flags)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const VarDecl, index: usize) ?*Node {
- var i = index;
-
- if (self.getTypeNode()) |type_node| {
- if (i < 1) return type_node;
- i -= 1;
- }
-
- if (self.getAlignNode()) |align_node| {
- if (i < 1) return align_node;
- i -= 1;
- }
-
- if (self.getSectionNode()) |section_node| {
- if (i < 1) return section_node;
- i -= 1;
- }
-
- if (self.getInitNode()) |init_node| {
- if (i < 1) return init_node;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const VarDecl) TokenIndex {
- if (self.getVisibToken()) |visib_token| return visib_token;
- if (self.getThreadLocalToken()) |thread_local_token| return thread_local_token;
- if (self.getComptimeToken()) |comptime_token| return comptime_token;
- if (self.getExternExportToken()) |extern_export_token| return extern_export_token;
- assert(self.getLibName() == null);
- return self.mut_token;
- }
-
- pub fn lastToken(self: *const VarDecl) TokenIndex {
- return self.semicolon_token;
- }
-
- fn sizeInBytes(trailer_flags: TrailerFlags) usize {
- return @sizeOf(VarDecl) + trailer_flags.sizeInBytes();
- }
+ pub const LocalVarDecl = struct {
+ type_node: Index,
+ align_node: Index,
};
- pub const Use = struct {
- base: Node = Node{ .tag = .Use },
- doc_comments: ?*DocComment,
- visib_token: ?TokenIndex,
- use_token: TokenIndex,
- expr: *Node,
- semicolon_token: TokenIndex,
-
- pub fn iterate(self: *const Use, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Use) TokenIndex {
- if (self.visib_token) |visib_token| return visib_token;
- return self.use_token;
- }
-
- pub fn lastToken(self: *const Use) TokenIndex {
- return self.semicolon_token;
- }
+ pub const ArrayTypeSentinel = struct {
+ elem_type: Index,
+ sentinel: Index,
};
- pub const ErrorSetDecl = struct {
- base: Node = Node{ .tag = .ErrorSetDecl },
- error_token: TokenIndex,
- rbrace_token: TokenIndex,
- decls_len: NodeIndex,
-
- /// After this the caller must initialize the decls list.
- pub fn alloc(allocator: *mem.Allocator, decls_len: NodeIndex) !*ErrorSetDecl {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(ErrorSetDecl), sizeInBytes(decls_len));
- return @ptrCast(*ErrorSetDecl, bytes.ptr);
- }
-
- pub fn free(self: *ErrorSetDecl, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.decls_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const ErrorSetDecl, index: usize) ?*Node {
- var i = index;
-
- if (i < self.decls_len) return self.declsConst()[i];
- i -= self.decls_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const ErrorSetDecl) TokenIndex {
- return self.error_token;
- }
-
- pub fn lastToken(self: *const ErrorSetDecl) TokenIndex {
- return self.rbrace_token;
- }
-
- pub fn decls(self: *ErrorSetDecl) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(ErrorSetDecl);
- return @ptrCast([*]*Node, decls_start)[0..self.decls_len];
- }
-
- pub fn declsConst(self: *const ErrorSetDecl) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ErrorSetDecl);
- return @ptrCast([*]const *Node, decls_start)[0..self.decls_len];
- }
-
- fn sizeInBytes(decls_len: NodeIndex) usize {
- return @sizeOf(ErrorSetDecl) + @sizeOf(*Node) * @as(usize, decls_len);
- }
+ pub const PtrType = struct {
+ sentinel: Index,
+ align_node: Index,
+ bit_range_start: Index,
+ bit_range_end: Index,
};
- /// The fields and decls Node pointers directly follow this struct in memory.
- pub const ContainerDecl = struct {
- base: Node = Node{ .tag = .ContainerDecl },
- kind_token: TokenIndex,
- layout_token: ?TokenIndex,
- lbrace_token: TokenIndex,
- rbrace_token: TokenIndex,
- fields_and_decls_len: NodeIndex,
- init_arg_expr: InitArg,
-
- pub const InitArg = union(enum) {
- None,
- Enum: ?*Node,
- Type: *Node,
- };
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, fields_and_decls_len: NodeIndex) !*ContainerDecl {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(ContainerDecl), sizeInBytes(fields_and_decls_len));
- return @ptrCast(*ContainerDecl, bytes.ptr);
- }
-
- pub fn free(self: *ContainerDecl, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.fields_and_decls_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const ContainerDecl, index: usize) ?*Node {
- var i = index;
-
- switch (self.init_arg_expr) {
- .Type => |t| {
- if (i < 1) return t;
- i -= 1;
- },
- .None, .Enum => {},
- }
-
- if (i < self.fields_and_decls_len) return self.fieldsAndDeclsConst()[i];
- i -= self.fields_and_decls_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const ContainerDecl) TokenIndex {
- if (self.layout_token) |layout_token| {
- return layout_token;
- }
- return self.kind_token;
- }
-
- pub fn lastToken(self: *const ContainerDecl) TokenIndex {
- return self.rbrace_token;
- }
-
- pub fn fieldsAndDecls(self: *ContainerDecl) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(ContainerDecl);
- return @ptrCast([*]*Node, decls_start)[0..self.fields_and_decls_len];
- }
-
- pub fn fieldsAndDeclsConst(self: *const ContainerDecl) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ContainerDecl);
- return @ptrCast([*]const *Node, decls_start)[0..self.fields_and_decls_len];
- }
+ pub const SliceType = struct {
+ sentinel: Index,
+ align_node: Index,
+ };
+ pub const SubRange = struct {
+ /// Index into sub_list.
+ start: Index,
+ /// Index into sub_list.
+ end: Index,
+ };
- fn sizeInBytes(fields_and_decls_len: NodeIndex) usize {
- return @sizeOf(ContainerDecl) + @sizeOf(*Node) * @as(usize, fields_and_decls_len);
- }
+ pub const If = struct {
+ then_expr: Index,
+ else_expr: Index,
};
pub const ContainerField = struct {
- base: Node = Node{ .tag = .ContainerField },
- doc_comments: ?*DocComment,
- comptime_token: ?TokenIndex,
- name_token: TokenIndex,
- type_expr: ?*Node,
- value_expr: ?*Node,
- align_expr: ?*Node,
+ value_expr: Index,
+ align_expr: Index,
+ };
- pub fn iterate(self: *const ContainerField, index: usize) ?*Node {
- var i = index;
+ pub const GlobalVarDecl = struct {
+ type_node: Index,
+ align_node: Index,
+ section_node: Index,
+ };
- if (self.type_expr) |type_expr| {
- if (i < 1) return type_expr;
- i -= 1;
- }
+ pub const Slice = struct {
+ start: Index,
+ end: Index,
+ sentinel: Index,
+ };
- if (self.align_expr) |align_expr| {
- if (i < 1) return align_expr;
- i -= 1;
- }
+ pub const While = struct {
+ continue_expr: Index,
+ then_expr: Index,
+ else_expr: Index,
+ };
- if (self.value_expr) |value_expr| {
- if (i < 1) return value_expr;
- i -= 1;
- }
+ pub const WhileCont = struct {
+ continue_expr: Index,
+ then_expr: Index,
+ };
- return null;
- }
-
- pub fn firstToken(self: *const ContainerField) TokenIndex {
- return self.comptime_token orelse self.name_token;
- }
-
- pub fn lastToken(self: *const ContainerField) TokenIndex {
- if (self.value_expr) |value_expr| {
- return value_expr.lastToken();
- }
- if (self.align_expr) |align_expr| {
- // The expression refers to what's inside the parenthesis, the
- // last token is the closing one
- return align_expr.lastToken() + 1;
- }
- if (self.type_expr) |type_expr| {
- return type_expr.lastToken();
- }
-
- return self.name_token;
- }
- };
-
- pub const ErrorTag = struct {
- base: Node = Node{ .tag = .ErrorTag },
- doc_comments: ?*DocComment,
- name_token: TokenIndex,
-
- pub fn iterate(self: *const ErrorTag, index: usize) ?*Node {
- var i = index;
-
- if (self.doc_comments) |comments| {
- if (i < 1) return &comments.base;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const ErrorTag) TokenIndex {
- return self.name_token;
- }
-
- pub fn lastToken(self: *const ErrorTag) TokenIndex {
- return self.name_token;
- }
- };
-
- pub const OneToken = struct {
- base: Node,
- token: TokenIndex,
-
- pub fn iterate(self: *const OneToken, index: usize) ?*Node {
- return null;
- }
-
- pub fn firstToken(self: *const OneToken) TokenIndex {
- return self.token;
- }
-
- pub fn lastToken(self: *const OneToken) TokenIndex {
- return self.token;
- }
- };
-
- /// The params are directly after the FnProto in memory.
- /// Next, each optional thing determined by a bit in `trailer_flags`.
- pub const FnProto = struct {
- base: Node = Node{ .tag = .FnProto },
- trailer_flags: TrailerFlags,
- fn_token: TokenIndex,
- params_len: NodeIndex,
- return_type: ReturnType,
-
- pub const TrailerFlags = std.meta.TrailerFlags(struct {
- doc_comments: *DocComment,
- body_node: *Node,
- lib_name: *Node, // populated if this is an extern declaration
- align_expr: *Node, // populated if align(A) is present
- section_expr: *Node, // populated if linksection(A) is present
- callconv_expr: *Node, // populated if callconv(A) is present
- visib_token: TokenIndex,
- name_token: TokenIndex,
- var_args_token: TokenIndex,
- extern_export_inline_token: TokenIndex,
- is_extern_prototype: void, // TODO: Remove once extern fn rewriting is
- is_async: void, // TODO: remove once async fn rewriting is
- });
-
- pub const RequiredFields = struct {
- fn_token: TokenIndex,
- params_len: NodeIndex,
- return_type: ReturnType,
- };
-
- pub const ReturnType = union(enum) {
- Explicit: *Node,
- InferErrorSet: *Node,
- Invalid: TokenIndex,
- };
-
- pub const ParamDecl = struct {
- doc_comments: ?*DocComment,
- comptime_token: ?TokenIndex,
- noalias_token: ?TokenIndex,
- name_token: ?TokenIndex,
- param_type: ParamType,
-
- pub const ParamType = union(enum) {
- any_type: *Node,
- type_expr: *Node,
- };
-
- pub fn iterate(self: *const ParamDecl, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) {
- switch (self.param_type) {
- .any_type, .type_expr => |node| return node,
- }
- }
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const ParamDecl) TokenIndex {
- if (self.comptime_token) |comptime_token| return comptime_token;
- if (self.noalias_token) |noalias_token| return noalias_token;
- if (self.name_token) |name_token| return name_token;
- switch (self.param_type) {
- .any_type, .type_expr => |node| return node.firstToken(),
- }
- }
-
- pub fn lastToken(self: *const ParamDecl) TokenIndex {
- switch (self.param_type) {
- .any_type, .type_expr => |node| return node.lastToken(),
- }
- }
- };
-
- /// For debugging purposes.
- pub fn dump(self: *const FnProto) void {
- const trailers_start = @alignCast(
- @alignOf(ParamDecl),
- @ptrCast([*]const u8, self) + @sizeOf(FnProto) + @sizeOf(ParamDecl) * self.params_len,
- );
- std.debug.print("{*} flags: {b} name_token: {s} {*} params_len: {d}\n", .{
- self,
- self.trailer_flags.bits,
- self.getNameToken(),
- self.trailer_flags.ptrConst(trailers_start, .name_token),
- self.params_len,
- });
- }
-
- pub fn getDocComments(self: *const FnProto) ?*DocComment {
- return self.getTrailer(.doc_comments);
- }
-
- pub fn setDocComments(self: *FnProto, value: *DocComment) void {
- self.setTrailer(.doc_comments, value);
- }
-
- pub fn getBodyNode(self: *const FnProto) ?*Node {
- return self.getTrailer(.body_node);
- }
-
- pub fn setBodyNode(self: *FnProto, value: *Node) void {
- self.setTrailer(.body_node, value);
- }
-
- pub fn getLibName(self: *const FnProto) ?*Node {
- return self.getTrailer(.lib_name);
- }
-
- pub fn setLibName(self: *FnProto, value: *Node) void {
- self.setTrailer(.lib_name, value);
- }
-
- pub fn getAlignExpr(self: *const FnProto) ?*Node {
- return self.getTrailer(.align_expr);
- }
-
- pub fn setAlignExpr(self: *FnProto, value: *Node) void {
- self.setTrailer(.align_expr, value);
- }
-
- pub fn getSectionExpr(self: *const FnProto) ?*Node {
- return self.getTrailer(.section_expr);
- }
-
- pub fn setSectionExpr(self: *FnProto, value: *Node) void {
- self.setTrailer(.section_expr, value);
- }
-
- pub fn getCallconvExpr(self: *const FnProto) ?*Node {
- return self.getTrailer(.callconv_expr);
- }
-
- pub fn setCallconvExpr(self: *FnProto, value: *Node) void {
- self.setTrailer(.callconv_expr, value);
- }
-
- pub fn getVisibToken(self: *const FnProto) ?TokenIndex {
- return self.getTrailer(.visib_token);
- }
-
- pub fn setVisibToken(self: *FnProto, value: TokenIndex) void {
- self.setTrailer(.visib_token, value);
- }
-
- pub fn getNameToken(self: *const FnProto) ?TokenIndex {
- return self.getTrailer(.name_token);
- }
-
- pub fn setNameToken(self: *FnProto, value: TokenIndex) void {
- self.setTrailer(.name_token, value);
- }
-
- pub fn getVarArgsToken(self: *const FnProto) ?TokenIndex {
- return self.getTrailer(.var_args_token);
- }
-
- pub fn setVarArgsToken(self: *FnProto, value: TokenIndex) void {
- self.setTrailer(.var_args_token, value);
- }
-
- pub fn getExternExportInlineToken(self: *const FnProto) ?TokenIndex {
- return self.getTrailer(.extern_export_inline_token);
- }
-
- pub fn setExternExportInlineToken(self: *FnProto, value: TokenIndex) void {
- self.setTrailer(.extern_export_inline_token, value);
- }
-
- pub fn getIsExternPrototype(self: *const FnProto) ?void {
- return self.getTrailer(.is_extern_prototype);
- }
-
- pub fn setIsExternPrototype(self: *FnProto, value: void) void {
- self.setTrailer(.is_extern_prototype, value);
- }
-
- pub fn getIsAsync(self: *const FnProto) ?void {
- return self.getTrailer(.is_async);
- }
-
- pub fn setIsAsync(self: *FnProto, value: void) void {
- self.setTrailer(.is_async, value);
- }
-
- fn getTrailer(self: *const FnProto, comptime field: TrailerFlags.FieldEnum) ?TrailerFlags.Field(field) {
- const trailers_start = @alignCast(
- @alignOf(ParamDecl),
- @ptrCast([*]const u8, self) + @sizeOf(FnProto) + @sizeOf(ParamDecl) * self.params_len,
- );
- return self.trailer_flags.get(trailers_start, field);
- }
-
- fn setTrailer(self: *FnProto, comptime field: TrailerFlags.FieldEnum, value: TrailerFlags.Field(field)) void {
- const trailers_start = @alignCast(
- @alignOf(ParamDecl),
- @ptrCast([*]u8, self) + @sizeOf(FnProto) + @sizeOf(ParamDecl) * self.params_len,
- );
- self.trailer_flags.set(trailers_start, field, value);
- }
-
- /// After this the caller must initialize the params list.
- pub fn create(allocator: *mem.Allocator, required: RequiredFields, trailers: TrailerFlags.InitStruct) !*FnProto {
- const trailer_flags = TrailerFlags.init(trailers);
- const bytes = try allocator.alignedAlloc(u8, @alignOf(FnProto), sizeInBytes(
- required.params_len,
- trailer_flags,
- ));
- const fn_proto = @ptrCast(*FnProto, bytes.ptr);
- fn_proto.* = .{
- .trailer_flags = trailer_flags,
- .fn_token = required.fn_token,
- .params_len = required.params_len,
- .return_type = required.return_type,
- };
- const trailers_start = @alignCast(
- @alignOf(ParamDecl),
- bytes.ptr + @sizeOf(FnProto) + @sizeOf(ParamDecl) * required.params_len,
- );
- trailer_flags.setMany(trailers_start, trailers);
- return fn_proto;
- }
-
- pub fn destroy(self: *FnProto, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len, self.trailer_flags)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const FnProto, index: usize) ?*Node {
- var i = index;
-
- if (self.getLibName()) |lib_name| {
- if (i < 1) return lib_name;
- i -= 1;
- }
-
- const params_len: usize = if (self.params_len == 0)
- 0
- else switch (self.paramsConst()[self.params_len - 1].param_type) {
- .any_type, .type_expr => self.params_len,
- };
- if (i < params_len) {
- switch (self.paramsConst()[i].param_type) {
- .any_type => |n| return n,
- .type_expr => |n| return n,
- }
- }
- i -= params_len;
-
- if (self.getAlignExpr()) |align_expr| {
- if (i < 1) return align_expr;
- i -= 1;
- }
-
- if (self.getSectionExpr()) |section_expr| {
- if (i < 1) return section_expr;
- i -= 1;
- }
-
- switch (self.return_type) {
- .Explicit, .InferErrorSet => |node| {
- if (i < 1) return node;
- i -= 1;
- },
- .Invalid => {},
- }
-
- if (self.getBodyNode()) |body_node| {
- if (i < 1) return body_node;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const FnProto) TokenIndex {
- if (self.getVisibToken()) |visib_token| return visib_token;
- if (self.getExternExportInlineToken()) |extern_export_inline_token| return extern_export_inline_token;
- assert(self.getLibName() == null);
- return self.fn_token;
- }
-
- pub fn lastToken(self: *const FnProto) TokenIndex {
- if (self.getBodyNode()) |body_node| return body_node.lastToken();
- switch (self.return_type) {
- .Explicit, .InferErrorSet => |node| return node.lastToken(),
- .Invalid => |tok| return tok,
- }
- }
-
- pub fn params(self: *FnProto) []ParamDecl {
- const params_start = @ptrCast([*]u8, self) + @sizeOf(FnProto);
- return @ptrCast([*]ParamDecl, params_start)[0..self.params_len];
- }
-
- pub fn paramsConst(self: *const FnProto) []const ParamDecl {
- const params_start = @ptrCast([*]const u8, self) + @sizeOf(FnProto);
- return @ptrCast([*]const ParamDecl, params_start)[0..self.params_len];
- }
-
- fn sizeInBytes(params_len: NodeIndex, trailer_flags: TrailerFlags) usize {
- return @sizeOf(FnProto) + @sizeOf(ParamDecl) * @as(usize, params_len) + trailer_flags.sizeInBytes();
- }
- };
-
- pub const AnyFrameType = struct {
- base: Node = Node{ .tag = .AnyFrameType },
- anyframe_token: TokenIndex,
- result: ?Result,
-
- pub const Result = struct {
- arrow_token: TokenIndex,
- return_type: *Node,
- };
-
- pub fn iterate(self: *const AnyFrameType, index: usize) ?*Node {
- var i = index;
-
- if (self.result) |result| {
- if (i < 1) return result.return_type;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const AnyFrameType) TokenIndex {
- return self.anyframe_token;
- }
-
- pub fn lastToken(self: *const AnyFrameType) TokenIndex {
- if (self.result) |result| return result.return_type.lastToken();
- return self.anyframe_token;
- }
- };
-
- /// The statements of the block follow Block directly in memory.
- pub const Block = struct {
- base: Node = Node{ .tag = .Block },
- statements_len: NodeIndex,
- lbrace: TokenIndex,
- rbrace: TokenIndex,
-
- /// After this the caller must initialize the statements list.
- pub fn alloc(allocator: *mem.Allocator, statements_len: NodeIndex) !*Block {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(Block), sizeInBytes(statements_len));
- return @ptrCast(*Block, bytes.ptr);
- }
-
- pub fn free(self: *Block, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.statements_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const Block, index: usize) ?*Node {
- var i = index;
-
- if (i < self.statements_len) return self.statementsConst()[i];
- i -= self.statements_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const Block) TokenIndex {
- return self.lbrace;
- }
-
- pub fn lastToken(self: *const Block) TokenIndex {
- return self.rbrace;
- }
-
- pub fn statements(self: *Block) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(Block);
- return @ptrCast([*]*Node, decls_start)[0..self.statements_len];
- }
-
- pub fn statementsConst(self: *const Block) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Block);
- return @ptrCast([*]const *Node, decls_start)[0..self.statements_len];
- }
-
- fn sizeInBytes(statements_len: NodeIndex) usize {
- return @sizeOf(Block) + @sizeOf(*Node) * @as(usize, statements_len);
- }
- };
-
- /// The statements of the block follow LabeledBlock directly in memory.
- pub const LabeledBlock = struct {
- base: Node = Node{ .tag = .LabeledBlock },
- statements_len: NodeIndex,
- lbrace: TokenIndex,
- rbrace: TokenIndex,
- label: TokenIndex,
-
- /// After this the caller must initialize the statements list.
- pub fn alloc(allocator: *mem.Allocator, statements_len: NodeIndex) !*LabeledBlock {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(LabeledBlock), sizeInBytes(statements_len));
- return @ptrCast(*LabeledBlock, bytes.ptr);
- }
-
- pub fn free(self: *LabeledBlock, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.statements_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const LabeledBlock, index: usize) ?*Node {
- var i = index;
-
- if (i < self.statements_len) return self.statementsConst()[i];
- i -= self.statements_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const LabeledBlock) TokenIndex {
- return self.label;
- }
-
- pub fn lastToken(self: *const LabeledBlock) TokenIndex {
- return self.rbrace;
- }
-
- pub fn statements(self: *LabeledBlock) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(LabeledBlock);
- return @ptrCast([*]*Node, decls_start)[0..self.statements_len];
- }
-
- pub fn statementsConst(self: *const LabeledBlock) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(LabeledBlock);
- return @ptrCast([*]const *Node, decls_start)[0..self.statements_len];
- }
-
- fn sizeInBytes(statements_len: NodeIndex) usize {
- return @sizeOf(LabeledBlock) + @sizeOf(*Node) * @as(usize, statements_len);
- }
- };
-
- pub const Defer = struct {
- base: Node = Node{ .tag = .Defer },
- defer_token: TokenIndex,
- payload: ?*Node,
- expr: *Node,
-
- pub fn iterate(self: *const Defer, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Defer) TokenIndex {
- return self.defer_token;
- }
-
- pub fn lastToken(self: *const Defer) TokenIndex {
- return self.expr.lastToken();
- }
- };
-
- pub const Comptime = struct {
- base: Node = Node{ .tag = .Comptime },
- doc_comments: ?*DocComment,
- comptime_token: TokenIndex,
- expr: *Node,
-
- pub fn iterate(self: *const Comptime, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Comptime) TokenIndex {
- return self.comptime_token;
- }
-
- pub fn lastToken(self: *const Comptime) TokenIndex {
- return self.expr.lastToken();
- }
- };
-
- pub const Nosuspend = struct {
- base: Node = Node{ .tag = .Nosuspend },
- nosuspend_token: TokenIndex,
- expr: *Node,
-
- pub fn iterate(self: *const Nosuspend, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Nosuspend) TokenIndex {
- return self.nosuspend_token;
- }
-
- pub fn lastToken(self: *const Nosuspend) TokenIndex {
- return self.expr.lastToken();
- }
- };
-
- pub const Payload = struct {
- base: Node = Node{ .tag = .Payload },
- lpipe: TokenIndex,
- error_symbol: *Node,
- rpipe: TokenIndex,
-
- pub fn iterate(self: *const Payload, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.error_symbol;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Payload) TokenIndex {
- return self.lpipe;
- }
-
- pub fn lastToken(self: *const Payload) TokenIndex {
- return self.rpipe;
- }
- };
-
- pub const PointerPayload = struct {
- base: Node = Node{ .tag = .PointerPayload },
- lpipe: TokenIndex,
- ptr_token: ?TokenIndex,
- value_symbol: *Node,
- rpipe: TokenIndex,
-
- pub fn iterate(self: *const PointerPayload, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.value_symbol;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const PointerPayload) TokenIndex {
- return self.lpipe;
- }
-
- pub fn lastToken(self: *const PointerPayload) TokenIndex {
- return self.rpipe;
- }
- };
-
- pub const PointerIndexPayload = struct {
- base: Node = Node{ .tag = .PointerIndexPayload },
- lpipe: TokenIndex,
- ptr_token: ?TokenIndex,
- value_symbol: *Node,
- index_symbol: ?*Node,
- rpipe: TokenIndex,
-
- pub fn iterate(self: *const PointerIndexPayload, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.value_symbol;
- i -= 1;
-
- if (self.index_symbol) |index_symbol| {
- if (i < 1) return index_symbol;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const PointerIndexPayload) TokenIndex {
- return self.lpipe;
- }
-
- pub fn lastToken(self: *const PointerIndexPayload) TokenIndex {
- return self.rpipe;
- }
- };
-
- pub const Else = struct {
- base: Node = Node{ .tag = .Else },
- else_token: TokenIndex,
- payload: ?*Node,
- body: *Node,
-
- pub fn iterate(self: *const Else, index: usize) ?*Node {
- var i = index;
-
- if (self.payload) |payload| {
- if (i < 1) return payload;
- i -= 1;
- }
-
- if (i < 1) return self.body;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Else) TokenIndex {
- return self.else_token;
- }
-
- pub fn lastToken(self: *const Else) TokenIndex {
- return self.body.lastToken();
- }
- };
-
- /// The cases node pointers are found in memory after Switch.
- /// They must be SwitchCase or SwitchElse nodes.
- pub const Switch = struct {
- base: Node = Node{ .tag = .Switch },
- switch_token: TokenIndex,
- rbrace: TokenIndex,
- cases_len: NodeIndex,
- expr: *Node,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, cases_len: NodeIndex) !*Switch {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(Switch), sizeInBytes(cases_len));
- return @ptrCast(*Switch, bytes.ptr);
- }
-
- pub fn free(self: *Switch, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.cases_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const Switch, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- if (i < self.cases_len) return self.casesConst()[i];
- i -= self.cases_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const Switch) TokenIndex {
- return self.switch_token;
- }
-
- pub fn lastToken(self: *const Switch) TokenIndex {
- return self.rbrace;
- }
-
- pub fn cases(self: *Switch) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(Switch);
- return @ptrCast([*]*Node, decls_start)[0..self.cases_len];
- }
-
- pub fn casesConst(self: *const Switch) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Switch);
- return @ptrCast([*]const *Node, decls_start)[0..self.cases_len];
- }
-
- fn sizeInBytes(cases_len: NodeIndex) usize {
- return @sizeOf(Switch) + @sizeOf(*Node) * @as(usize, cases_len);
- }
- };
-
- /// Items sub-nodes appear in memory directly following SwitchCase.
- pub const SwitchCase = struct {
- base: Node = Node{ .tag = .SwitchCase },
- arrow_token: TokenIndex,
- payload: ?*Node,
- expr: *Node,
- items_len: NodeIndex,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, items_len: NodeIndex) !*SwitchCase {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(SwitchCase), sizeInBytes(items_len));
- return @ptrCast(*SwitchCase, bytes.ptr);
- }
-
- pub fn free(self: *SwitchCase, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.items_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const SwitchCase, index: usize) ?*Node {
- var i = index;
-
- if (i < self.items_len) return self.itemsConst()[i];
- i -= self.items_len;
-
- if (self.payload) |payload| {
- if (i < 1) return payload;
- i -= 1;
- }
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const SwitchCase) TokenIndex {
- return self.itemsConst()[0].firstToken();
- }
-
- pub fn lastToken(self: *const SwitchCase) TokenIndex {
- return self.expr.lastToken();
- }
-
- pub fn items(self: *SwitchCase) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(SwitchCase);
- return @ptrCast([*]*Node, decls_start)[0..self.items_len];
- }
-
- pub fn itemsConst(self: *const SwitchCase) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(SwitchCase);
- return @ptrCast([*]const *Node, decls_start)[0..self.items_len];
- }
-
- fn sizeInBytes(items_len: NodeIndex) usize {
- return @sizeOf(SwitchCase) + @sizeOf(*Node) * @as(usize, items_len);
- }
- };
-
- pub const SwitchElse = struct {
- base: Node = Node{ .tag = .SwitchElse },
- token: TokenIndex,
-
- pub fn iterate(self: *const SwitchElse, index: usize) ?*Node {
- return null;
- }
-
- pub fn firstToken(self: *const SwitchElse) TokenIndex {
- return self.token;
- }
-
- pub fn lastToken(self: *const SwitchElse) TokenIndex {
- return self.token;
- }
- };
-
- pub const While = struct {
- base: Node = Node{ .tag = .While },
- label: ?TokenIndex,
- inline_token: ?TokenIndex,
- while_token: TokenIndex,
- condition: *Node,
- payload: ?*Node,
- continue_expr: ?*Node,
- body: *Node,
- @"else": ?*Else,
-
- pub fn iterate(self: *const While, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.condition;
- i -= 1;
-
- if (self.payload) |payload| {
- if (i < 1) return payload;
- i -= 1;
- }
-
- if (self.continue_expr) |continue_expr| {
- if (i < 1) return continue_expr;
- i -= 1;
- }
-
- if (i < 1) return self.body;
- i -= 1;
-
- if (self.@"else") |@"else"| {
- if (i < 1) return &@"else".base;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const While) TokenIndex {
- if (self.label) |label| {
- return label;
- }
-
- if (self.inline_token) |inline_token| {
- return inline_token;
- }
-
- return self.while_token;
- }
-
- pub fn lastToken(self: *const While) TokenIndex {
- if (self.@"else") |@"else"| {
- return @"else".body.lastToken();
- }
-
- return self.body.lastToken();
- }
- };
-
- pub const For = struct {
- base: Node = Node{ .tag = .For },
- label: ?TokenIndex,
- inline_token: ?TokenIndex,
- for_token: TokenIndex,
- array_expr: *Node,
- payload: *Node,
- body: *Node,
- @"else": ?*Else,
-
- pub fn iterate(self: *const For, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.array_expr;
- i -= 1;
-
- if (i < 1) return self.payload;
- i -= 1;
-
- if (i < 1) return self.body;
- i -= 1;
-
- if (self.@"else") |@"else"| {
- if (i < 1) return &@"else".base;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const For) TokenIndex {
- if (self.label) |label| {
- return label;
- }
-
- if (self.inline_token) |inline_token| {
- return inline_token;
- }
-
- return self.for_token;
- }
-
- pub fn lastToken(self: *const For) TokenIndex {
- if (self.@"else") |@"else"| {
- return @"else".body.lastToken();
- }
-
- return self.body.lastToken();
- }
- };
-
- pub const If = struct {
- base: Node = Node{ .tag = .If },
- if_token: TokenIndex,
- condition: *Node,
- payload: ?*Node,
- body: *Node,
- @"else": ?*Else,
-
- pub fn iterate(self: *const If, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.condition;
- i -= 1;
-
- if (self.payload) |payload| {
- if (i < 1) return payload;
- i -= 1;
- }
-
- if (i < 1) return self.body;
- i -= 1;
-
- if (self.@"else") |@"else"| {
- if (i < 1) return &@"else".base;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const If) TokenIndex {
- return self.if_token;
- }
-
- pub fn lastToken(self: *const If) TokenIndex {
- if (self.@"else") |@"else"| {
- return @"else".body.lastToken();
- }
-
- return self.body.lastToken();
- }
- };
-
- pub const Catch = struct {
- base: Node = Node{ .tag = .Catch },
- op_token: TokenIndex,
- lhs: *Node,
- rhs: *Node,
- payload: ?*Node,
-
- pub fn iterate(self: *const Catch, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (self.payload) |payload| {
- if (i < 1) return payload;
- i -= 1;
- }
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Catch) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const Catch) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const SimpleInfixOp = struct {
- base: Node,
- op_token: TokenIndex,
- lhs: *Node,
- rhs: *Node,
-
- pub fn iterate(self: *const SimpleInfixOp, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const SimpleInfixOp) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const SimpleInfixOp) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const SimplePrefixOp = struct {
- base: Node,
- op_token: TokenIndex,
- rhs: *Node,
-
- const Self = @This();
-
- pub fn iterate(self: *const Self, index: usize) ?*Node {
- if (index == 0) return self.rhs;
- return null;
- }
-
- pub fn firstToken(self: *const Self) TokenIndex {
- return self.op_token;
- }
-
- pub fn lastToken(self: *const Self) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const ArrayType = struct {
- base: Node = Node{ .tag = .ArrayType },
- op_token: TokenIndex,
- rhs: *Node,
- len_expr: *Node,
-
- pub fn iterate(self: *const ArrayType, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.len_expr;
- i -= 1;
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const ArrayType) TokenIndex {
- return self.op_token;
- }
-
- pub fn lastToken(self: *const ArrayType) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const ArrayTypeSentinel = struct {
- base: Node = Node{ .tag = .ArrayTypeSentinel },
- op_token: TokenIndex,
- rhs: *Node,
- len_expr: *Node,
- sentinel: *Node,
-
- pub fn iterate(self: *const ArrayTypeSentinel, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.len_expr;
- i -= 1;
-
- if (i < 1) return self.sentinel;
- i -= 1;
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const ArrayTypeSentinel) TokenIndex {
- return self.op_token;
- }
-
- pub fn lastToken(self: *const ArrayTypeSentinel) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const PtrType = struct {
- base: Node = Node{ .tag = .PtrType },
- op_token: TokenIndex,
- rhs: *Node,
- /// TODO Add a u8 flags field to Node where it would otherwise be padding, and each bit represents
- /// one of these possibly-null things. Then we have them directly follow the PtrType in memory.
- ptr_info: PtrInfo = .{},
-
- pub fn iterate(self: *const PtrType, index: usize) ?*Node {
- var i = index;
-
- if (self.ptr_info.sentinel) |sentinel| {
- if (i < 1) return sentinel;
- i -= 1;
- }
-
- if (self.ptr_info.align_info) |align_info| {
- if (i < 1) return align_info.node;
- i -= 1;
- }
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const PtrType) TokenIndex {
- return self.op_token;
- }
-
- pub fn lastToken(self: *const PtrType) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const SliceType = struct {
- base: Node = Node{ .tag = .SliceType },
- op_token: TokenIndex,
- rhs: *Node,
- /// TODO Add a u8 flags field to Node where it would otherwise be padding, and each bit represents
- /// one of these possibly-null things. Then we have them directly follow the SliceType in memory.
- ptr_info: PtrInfo = .{},
-
- pub fn iterate(self: *const SliceType, index: usize) ?*Node {
- var i = index;
-
- if (self.ptr_info.sentinel) |sentinel| {
- if (i < 1) return sentinel;
- i -= 1;
- }
-
- if (self.ptr_info.align_info) |align_info| {
- if (i < 1) return align_info.node;
- i -= 1;
- }
-
- if (i < 1) return self.rhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const SliceType) TokenIndex {
- return self.op_token;
- }
-
- pub fn lastToken(self: *const SliceType) TokenIndex {
- return self.rhs.lastToken();
- }
- };
-
- pub const FieldInitializer = struct {
- base: Node = Node{ .tag = .FieldInitializer },
- period_token: TokenIndex,
- name_token: TokenIndex,
- expr: *Node,
-
- pub fn iterate(self: *const FieldInitializer, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const FieldInitializer) TokenIndex {
- return self.period_token;
- }
-
- pub fn lastToken(self: *const FieldInitializer) TokenIndex {
- return self.expr.lastToken();
- }
- };
-
- /// Elements occur directly in memory after ArrayInitializer.
- pub const ArrayInitializer = struct {
- base: Node = Node{ .tag = .ArrayInitializer },
- rtoken: TokenIndex,
- list_len: NodeIndex,
- lhs: *Node,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializer {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializer), sizeInBytes(list_len));
- return @ptrCast(*ArrayInitializer, bytes.ptr);
- }
-
- pub fn free(self: *ArrayInitializer, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const ArrayInitializer, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < self.list_len) return self.listConst()[i];
- i -= self.list_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const ArrayInitializer) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const ArrayInitializer) TokenIndex {
- return self.rtoken;
- }
-
- pub fn list(self: *ArrayInitializer) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializer);
- return @ptrCast([*]*Node, decls_start)[0..self.list_len];
- }
-
- pub fn listConst(self: *const ArrayInitializer) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializer);
- return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
- }
-
- fn sizeInBytes(list_len: NodeIndex) usize {
- return @sizeOf(ArrayInitializer) + @sizeOf(*Node) * @as(usize, list_len);
- }
- };
-
- /// Elements occur directly in memory after ArrayInitializerDot.
- pub const ArrayInitializerDot = struct {
- base: Node = Node{ .tag = .ArrayInitializerDot },
- dot: TokenIndex,
- rtoken: TokenIndex,
- list_len: NodeIndex,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializerDot {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializerDot), sizeInBytes(list_len));
- return @ptrCast(*ArrayInitializerDot, bytes.ptr);
- }
-
- pub fn free(self: *ArrayInitializerDot, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const ArrayInitializerDot, index: usize) ?*Node {
- var i = index;
-
- if (i < self.list_len) return self.listConst()[i];
- i -= self.list_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const ArrayInitializerDot) TokenIndex {
- return self.dot;
- }
-
- pub fn lastToken(self: *const ArrayInitializerDot) TokenIndex {
- return self.rtoken;
- }
-
- pub fn list(self: *ArrayInitializerDot) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializerDot);
- return @ptrCast([*]*Node, decls_start)[0..self.list_len];
- }
-
- pub fn listConst(self: *const ArrayInitializerDot) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializerDot);
- return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
- }
-
- fn sizeInBytes(list_len: NodeIndex) usize {
- return @sizeOf(ArrayInitializerDot) + @sizeOf(*Node) * @as(usize, list_len);
- }
- };
-
- /// Elements occur directly in memory after StructInitializer.
- pub const StructInitializer = struct {
- base: Node = Node{ .tag = .StructInitializer },
- rtoken: TokenIndex,
- list_len: NodeIndex,
- lhs: *Node,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializer {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializer), sizeInBytes(list_len));
- return @ptrCast(*StructInitializer, bytes.ptr);
- }
-
- pub fn free(self: *StructInitializer, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const StructInitializer, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < self.list_len) return self.listConst()[i];
- i -= self.list_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const StructInitializer) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const StructInitializer) TokenIndex {
- return self.rtoken;
- }
-
- pub fn list(self: *StructInitializer) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializer);
- return @ptrCast([*]*Node, decls_start)[0..self.list_len];
- }
-
- pub fn listConst(self: *const StructInitializer) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializer);
- return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
- }
-
- fn sizeInBytes(list_len: NodeIndex) usize {
- return @sizeOf(StructInitializer) + @sizeOf(*Node) * @as(usize, list_len);
- }
- };
-
- /// Elements occur directly in memory after StructInitializerDot.
- pub const StructInitializerDot = struct {
- base: Node = Node{ .tag = .StructInitializerDot },
- dot: TokenIndex,
- rtoken: TokenIndex,
- list_len: NodeIndex,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializerDot {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializerDot), sizeInBytes(list_len));
- return @ptrCast(*StructInitializerDot, bytes.ptr);
- }
-
- pub fn free(self: *StructInitializerDot, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const StructInitializerDot, index: usize) ?*Node {
- var i = index;
-
- if (i < self.list_len) return self.listConst()[i];
- i -= self.list_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const StructInitializerDot) TokenIndex {
- return self.dot;
- }
-
- pub fn lastToken(self: *const StructInitializerDot) TokenIndex {
- return self.rtoken;
- }
-
- pub fn list(self: *StructInitializerDot) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializerDot);
- return @ptrCast([*]*Node, decls_start)[0..self.list_len];
- }
-
- pub fn listConst(self: *const StructInitializerDot) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializerDot);
- return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
- }
-
- fn sizeInBytes(list_len: NodeIndex) usize {
- return @sizeOf(StructInitializerDot) + @sizeOf(*Node) * @as(usize, list_len);
- }
- };
-
- /// Parameter nodes directly follow Call in memory.
- pub const Call = struct {
- base: Node = Node{ .tag = .Call },
- rtoken: TokenIndex,
- lhs: *Node,
- params_len: NodeIndex,
- async_token: ?TokenIndex,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, params_len: NodeIndex) !*Call {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(Call), sizeInBytes(params_len));
- return @ptrCast(*Call, bytes.ptr);
- }
-
- pub fn free(self: *Call, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const Call, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < self.params_len) return self.paramsConst()[i];
- i -= self.params_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const Call) TokenIndex {
- if (self.async_token) |async_token| return async_token;
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const Call) TokenIndex {
- return self.rtoken;
- }
-
- pub fn params(self: *Call) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(Call);
- return @ptrCast([*]*Node, decls_start)[0..self.params_len];
- }
-
- pub fn paramsConst(self: *const Call) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Call);
- return @ptrCast([*]const *Node, decls_start)[0..self.params_len];
- }
-
- fn sizeInBytes(params_len: NodeIndex) usize {
- return @sizeOf(Call) + @sizeOf(*Node) * @as(usize, params_len);
- }
- };
-
- pub const ArrayAccess = struct {
- base: Node = Node{ .tag = .ArrayAccess },
- rtoken: TokenIndex,
- lhs: *Node,
- index_expr: *Node,
-
- pub fn iterate(self: *const ArrayAccess, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < 1) return self.index_expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const ArrayAccess) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const ArrayAccess) TokenIndex {
- return self.rtoken;
- }
- };
-
- pub const SimpleSuffixOp = struct {
- base: Node,
- rtoken: TokenIndex,
- lhs: *Node,
-
- pub fn iterate(self: *const SimpleSuffixOp, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const SimpleSuffixOp) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const SimpleSuffixOp) TokenIndex {
- return self.rtoken;
- }
- };
-
- pub const Slice = struct {
- base: Node = Node{ .tag = .Slice },
- rtoken: TokenIndex,
- lhs: *Node,
- start: *Node,
- end: ?*Node,
- sentinel: ?*Node,
-
- pub fn iterate(self: *const Slice, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.lhs;
- i -= 1;
-
- if (i < 1) return self.start;
- i -= 1;
-
- if (self.end) |end| {
- if (i < 1) return end;
- i -= 1;
- }
- if (self.sentinel) |sentinel| {
- if (i < 1) return sentinel;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const Slice) TokenIndex {
- return self.lhs.firstToken();
- }
-
- pub fn lastToken(self: *const Slice) TokenIndex {
- return self.rtoken;
- }
- };
-
- pub const GroupedExpression = struct {
- base: Node = Node{ .tag = .GroupedExpression },
- lparen: TokenIndex,
- expr: *Node,
- rparen: TokenIndex,
-
- pub fn iterate(self: *const GroupedExpression, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const GroupedExpression) TokenIndex {
- return self.lparen;
- }
-
- pub fn lastToken(self: *const GroupedExpression) TokenIndex {
- return self.rparen;
- }
- };
-
- /// Trailed in memory by possibly many things, with each optional thing
- /// determined by a bit in `trailer_flags`.
- /// Can be: return, break, continue
- pub const ControlFlowExpression = struct {
- base: Node,
- trailer_flags: TrailerFlags,
- ltoken: TokenIndex,
-
- pub const TrailerFlags = std.meta.TrailerFlags(struct {
- rhs: *Node,
- label: TokenIndex,
- });
-
- pub const RequiredFields = struct {
- tag: Tag,
- ltoken: TokenIndex,
- };
-
- pub fn getRHS(self: *const ControlFlowExpression) ?*Node {
- return self.getTrailer(.rhs);
- }
-
- pub fn setRHS(self: *ControlFlowExpression, value: *Node) void {
- self.setTrailer(.rhs, value);
- }
-
- pub fn getLabel(self: *const ControlFlowExpression) ?TokenIndex {
- return self.getTrailer(.label);
- }
-
- pub fn setLabel(self: *ControlFlowExpression, value: TokenIndex) void {
- self.setTrailer(.label, value);
- }
-
- fn getTrailer(self: *const ControlFlowExpression, comptime field: TrailerFlags.FieldEnum) ?TrailerFlags.Field(field) {
- const trailers_start = @ptrCast([*]const u8, self) + @sizeOf(ControlFlowExpression);
- return self.trailer_flags.get(trailers_start, field);
- }
-
- fn setTrailer(self: *ControlFlowExpression, comptime field: TrailerFlags.FieldEnum, value: TrailerFlags.Field(field)) void {
- const trailers_start = @ptrCast([*]u8, self) + @sizeOf(ControlFlowExpression);
- self.trailer_flags.set(trailers_start, field, value);
- }
-
- pub fn create(allocator: *mem.Allocator, required: RequiredFields, trailers: TrailerFlags.InitStruct) !*ControlFlowExpression {
- const trailer_flags = TrailerFlags.init(trailers);
- const bytes = try allocator.alignedAlloc(u8, @alignOf(ControlFlowExpression), sizeInBytes(trailer_flags));
- const ctrl_flow_expr = @ptrCast(*ControlFlowExpression, bytes.ptr);
- ctrl_flow_expr.* = .{
- .base = .{ .tag = required.tag },
- .trailer_flags = trailer_flags,
- .ltoken = required.ltoken,
- };
- const trailers_start = bytes.ptr + @sizeOf(ControlFlowExpression);
- trailer_flags.setMany(trailers_start, trailers);
- return ctrl_flow_expr;
- }
-
- pub fn destroy(self: *ControlFlowExpression, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.trailer_flags)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const ControlFlowExpression, index: usize) ?*Node {
- var i = index;
-
- if (self.getRHS()) |rhs| {
- if (i < 1) return rhs;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const ControlFlowExpression) TokenIndex {
- return self.ltoken;
- }
-
- pub fn lastToken(self: *const ControlFlowExpression) TokenIndex {
- if (self.getRHS()) |rhs| {
- return rhs.lastToken();
- }
-
- if (self.getLabel()) |label| {
- return label;
- }
-
- return self.ltoken;
- }
-
- fn sizeInBytes(trailer_flags: TrailerFlags) usize {
- return @sizeOf(ControlFlowExpression) + trailer_flags.sizeInBytes();
- }
- };
-
- pub const Suspend = struct {
- base: Node = Node{ .tag = .Suspend },
- suspend_token: TokenIndex,
- body: ?*Node,
-
- pub fn iterate(self: *const Suspend, index: usize) ?*Node {
- var i = index;
-
- if (self.body) |body| {
- if (i < 1) return body;
- i -= 1;
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const Suspend) TokenIndex {
- return self.suspend_token;
- }
-
- pub fn lastToken(self: *const Suspend) TokenIndex {
- if (self.body) |body| {
- return body.lastToken();
- }
-
- return self.suspend_token;
- }
- };
-
- pub const EnumLiteral = struct {
- base: Node = Node{ .tag = .EnumLiteral },
- dot: TokenIndex,
- name: TokenIndex,
-
- pub fn iterate(self: *const EnumLiteral, index: usize) ?*Node {
- return null;
- }
-
- pub fn firstToken(self: *const EnumLiteral) TokenIndex {
- return self.dot;
- }
-
- pub fn lastToken(self: *const EnumLiteral) TokenIndex {
- return self.name;
- }
- };
-
- /// Parameters are in memory following BuiltinCall.
- pub const BuiltinCall = struct {
- base: Node = Node{ .tag = .BuiltinCall },
- params_len: NodeIndex,
- builtin_token: TokenIndex,
- rparen_token: TokenIndex,
-
- /// After this the caller must initialize the fields_and_decls list.
- pub fn alloc(allocator: *mem.Allocator, params_len: NodeIndex) !*BuiltinCall {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(BuiltinCall), sizeInBytes(params_len));
- return @ptrCast(*BuiltinCall, bytes.ptr);
- }
-
- pub fn free(self: *BuiltinCall, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const BuiltinCall, index: usize) ?*Node {
- var i = index;
-
- if (i < self.params_len) return self.paramsConst()[i];
- i -= self.params_len;
-
- return null;
- }
-
- pub fn firstToken(self: *const BuiltinCall) TokenIndex {
- return self.builtin_token;
- }
-
- pub fn lastToken(self: *const BuiltinCall) TokenIndex {
- return self.rparen_token;
- }
-
- pub fn params(self: *BuiltinCall) []*Node {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(BuiltinCall);
- return @ptrCast([*]*Node, decls_start)[0..self.params_len];
- }
-
- pub fn paramsConst(self: *const BuiltinCall) []const *Node {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(BuiltinCall);
- return @ptrCast([*]const *Node, decls_start)[0..self.params_len];
- }
-
- fn sizeInBytes(params_len: NodeIndex) usize {
- return @sizeOf(BuiltinCall) + @sizeOf(*Node) * @as(usize, params_len);
- }
- };
-
- /// The string literal tokens appear directly in memory after MultilineStringLiteral.
- pub const MultilineStringLiteral = struct {
- base: Node = Node{ .tag = .MultilineStringLiteral },
- lines_len: TokenIndex,
-
- /// After this the caller must initialize the lines list.
- pub fn alloc(allocator: *mem.Allocator, lines_len: NodeIndex) !*MultilineStringLiteral {
- const bytes = try allocator.alignedAlloc(u8, @alignOf(MultilineStringLiteral), sizeInBytes(lines_len));
- return @ptrCast(*MultilineStringLiteral, bytes.ptr);
- }
-
- pub fn free(self: *MultilineStringLiteral, allocator: *mem.Allocator) void {
- const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.lines_len)];
- allocator.free(bytes);
- }
-
- pub fn iterate(self: *const MultilineStringLiteral, index: usize) ?*Node {
- return null;
- }
-
- pub fn firstToken(self: *const MultilineStringLiteral) TokenIndex {
- return self.linesConst()[0];
- }
-
- pub fn lastToken(self: *const MultilineStringLiteral) TokenIndex {
- return self.linesConst()[self.lines_len - 1];
- }
-
- pub fn lines(self: *MultilineStringLiteral) []TokenIndex {
- const decls_start = @ptrCast([*]u8, self) + @sizeOf(MultilineStringLiteral);
- return @ptrCast([*]TokenIndex, decls_start)[0..self.lines_len];
- }
-
- pub fn linesConst(self: *const MultilineStringLiteral) []const TokenIndex {
- const decls_start = @ptrCast([*]const u8, self) + @sizeOf(MultilineStringLiteral);
- return @ptrCast([*]const TokenIndex, decls_start)[0..self.lines_len];
- }
-
- fn sizeInBytes(lines_len: NodeIndex) usize {
- return @sizeOf(MultilineStringLiteral) + @sizeOf(TokenIndex) * @as(usize, lines_len);
- }
- };
-
- pub const Asm = struct {
- base: Node = Node{ .tag = .Asm },
- asm_token: TokenIndex,
- rparen: TokenIndex,
- volatile_token: ?TokenIndex,
- template: *Node,
- outputs: []Output,
- inputs: []Input,
- /// A clobber node must be a StringLiteral or MultilineStringLiteral.
- clobbers: []*Node,
-
- pub const Output = struct {
- lbracket: TokenIndex,
- symbolic_name: *Node,
- constraint: *Node,
- kind: Kind,
- rparen: TokenIndex,
-
- pub const Kind = union(enum) {
- Variable: *OneToken,
- Return: *Node,
- };
-
- pub fn iterate(self: *const Output, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.symbolic_name;
- i -= 1;
-
- if (i < 1) return self.constraint;
- i -= 1;
-
- switch (self.kind) {
- .Variable => |variable_name| {
- if (i < 1) return &variable_name.base;
- i -= 1;
- },
- .Return => |return_type| {
- if (i < 1) return return_type;
- i -= 1;
- },
- }
-
- return null;
- }
-
- pub fn firstToken(self: *const Output) TokenIndex {
- return self.lbracket;
- }
-
- pub fn lastToken(self: *const Output) TokenIndex {
- return self.rparen;
- }
- };
-
- pub const Input = struct {
- lbracket: TokenIndex,
- symbolic_name: *Node,
- constraint: *Node,
- expr: *Node,
- rparen: TokenIndex,
-
- pub fn iterate(self: *const Input, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.symbolic_name;
- i -= 1;
-
- if (i < 1) return self.constraint;
- i -= 1;
-
- if (i < 1) return self.expr;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const Input) TokenIndex {
- return self.lbracket;
- }
-
- pub fn lastToken(self: *const Input) TokenIndex {
- return self.rparen;
- }
- };
-
- pub fn iterate(self: *const Asm, index: usize) ?*Node {
- var i = index;
-
- if (i < self.outputs.len * 3) switch (i % 3) {
- 0 => return self.outputs[i / 3].symbolic_name,
- 1 => return self.outputs[i / 3].constraint,
- 2 => switch (self.outputs[i / 3].kind) {
- .Variable => |variable_name| return &variable_name.base,
- .Return => |return_type| return return_type,
- },
- else => unreachable,
- };
- i -= self.outputs.len * 3;
-
- if (i < self.inputs.len * 3) switch (i % 3) {
- 0 => return self.inputs[i / 3].symbolic_name,
- 1 => return self.inputs[i / 3].constraint,
- 2 => return self.inputs[i / 3].expr,
- else => unreachable,
- };
- i -= self.inputs.len * 3;
-
- return null;
- }
-
- pub fn firstToken(self: *const Asm) TokenIndex {
- return self.asm_token;
- }
-
- pub fn lastToken(self: *const Asm) TokenIndex {
- return self.rparen;
- }
- };
-
- /// TODO remove from the Node base struct
- /// TODO actually maybe remove entirely in favor of iterating backward from Node.firstToken()
- /// and forwards to find same-line doc comments.
- pub const DocComment = struct {
- base: Node = Node{ .tag = .DocComment },
- /// Points to the first doc comment token. API users are expected to iterate over the
- /// tokens array, looking for more doc comments, ignoring line comments, and stopping
- /// at the first other token.
- first_line: TokenIndex,
-
- pub fn iterate(self: *const DocComment, index: usize) ?*Node {
- return null;
- }
-
- pub fn firstToken(self: *const DocComment) TokenIndex {
- return self.first_line;
- }
-
- /// Returns the first doc comment line. Be careful, this may not be the desired behavior,
- /// which would require the tokens array.
- pub fn lastToken(self: *const DocComment) TokenIndex {
- return self.first_line;
- }
- };
-
- pub const TestDecl = struct {
- base: Node = Node{ .tag = .TestDecl },
- doc_comments: ?*DocComment,
- test_token: TokenIndex,
- name: ?*Node,
- body_node: *Node,
-
- pub fn iterate(self: *const TestDecl, index: usize) ?*Node {
- var i = index;
-
- if (i < 1) return self.body_node;
- i -= 1;
-
- return null;
- }
-
- pub fn firstToken(self: *const TestDecl) TokenIndex {
- return self.test_token;
- }
-
- pub fn lastToken(self: *const TestDecl) TokenIndex {
- return self.body_node.lastToken();
- }
+ pub const FnProtoOne = struct {
+ /// Populated if there is exactly 1 parameter. Otherwise there are 0 parameters.
+ param: Index,
+ /// Populated if align(A) is present.
+ align_expr: Index,
+ /// Populated if linksection(A) is present.
+ section_expr: Index,
+ /// Populated if callconv(A) is present.
+ callconv_expr: Index,
};
-};
-
-pub const PtrInfo = struct {
- allowzero_token: ?TokenIndex = null,
- align_info: ?Align = null,
- const_token: ?TokenIndex = null,
- volatile_token: ?TokenIndex = null,
- sentinel: ?*Node = null,
-
- pub const Align = struct {
- node: *Node,
- bit_range: ?BitRange = null,
- pub const BitRange = struct {
- start: *Node,
- end: *Node,
- };
+ pub const FnProto = struct {
+ params_start: Index,
+ params_end: Index,
+ /// Populated if align(A) is present.
+ align_expr: Index,
+ /// Populated if linksection(A) is present.
+ section_expr: Index,
+ /// Populated if callconv(A) is present.
+ callconv_expr: Index,
};
};
-
-test "iterate" {
- var root = Node.Root{
- .base = Node{ .tag = Node.Tag.Root },
- .decls_len = 0,
- .eof_token = 0,
- };
- var base = &root.base;
- testing.expect(base.iterate(0) == null);
-}
lib/std/zig/parse.zig
@@ -11,85 +11,138 @@ const Node = ast.Node;
const Tree = ast.Tree;
const AstError = ast.Error;
const TokenIndex = ast.TokenIndex;
-const NodeIndex = ast.NodeIndex;
const Token = std.zig.Token;
pub const Error = error{ParseError} || Allocator.Error;
/// Result should be freed with tree.deinit() when there are
/// no more references to any of the tokens or nodes.
-pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
- var token_ids = std.ArrayList(Token.Id).init(gpa);
- defer token_ids.deinit();
- var token_locs = std.ArrayList(Token.Loc).init(gpa);
- defer token_locs.deinit();
+pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {
+ var tokens = ast.TokenList{};
+ defer tokens.deinit(gpa);
// Empirically, the zig std lib has an 8:1 ratio of source bytes to token count.
const estimated_token_count = source.len / 8;
- try token_ids.ensureCapacity(estimated_token_count);
- try token_locs.ensureCapacity(estimated_token_count);
+ try tokens.ensureCapacity(gpa, estimated_token_count);
var tokenizer = std.zig.Tokenizer.init(source);
while (true) {
const token = tokenizer.next();
- try token_ids.append(token.id);
- try token_locs.append(token.loc);
- if (token.id == .Eof) break;
+ if (token.tag == .LineComment) continue;
+ try tokens.append(gpa, .{
+ .tag = token.tag,
+ .start = @intCast(u32, token.loc.start),
+ });
+ if (token.tag == .Eof) break;
}
var parser: Parser = .{
.source = source,
- .arena = std.heap.ArenaAllocator.init(gpa),
.gpa = gpa,
- .token_ids = token_ids.items,
- .token_locs = token_locs.items,
+ .token_tags = tokens.items(.tag),
+ .token_starts = tokens.items(.start),
.errors = .{},
+ .nodes = .{},
+ .extra_data = .{},
.tok_i = 0,
};
defer parser.errors.deinit(gpa);
- errdefer parser.arena.deinit();
-
- while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1;
-
- const root_node = try parser.parseRoot();
+ defer parser.nodes.deinit(gpa);
+ defer parser.extra_data.deinit(gpa);
+
+ // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes.
+ // Make sure at least 1 so we can use appendAssumeCapacity on the root node below.
+ const estimated_node_count = (tokens.len + 2) / 2;
+ try parser.nodes.ensureCapacity(gpa, estimated_node_count);
+
+ // Root node must be index 0.
+ // Root <- skip ContainerMembers eof
+ parser.nodes.appendAssumeCapacity(.{
+ .tag = .Root,
+ .main_token = 0,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
+ const root_decls = try parser.parseContainerMembers(true);
+ // parseContainerMembers will try to skip as much
+ // invalid tokens as it can, so we are now at EOF.
+ assert(parser.token_tags[parser.tok_i] == .Eof);
+ parser.nodes.items(.data)[0] = .{
+ .lhs = root_decls.start,
+ .rhs = root_decls.end,
+ };
- const tree = try parser.arena.allocator.create(Tree);
- tree.* = .{
- .gpa = gpa,
+ // TODO experiment with compacting the MultiArrayList slices here
+ return Tree{
.source = source,
- .token_ids = token_ids.toOwnedSlice(),
- .token_locs = token_locs.toOwnedSlice(),
+ .tokens = tokens.toOwnedSlice(),
+ .nodes = parser.nodes.toOwnedSlice(),
+ .extra_data = parser.extra_data.toOwnedSlice(gpa),
.errors = parser.errors.toOwnedSlice(gpa),
- .root_node = root_node,
- .arena = parser.arena.state,
};
- return tree;
}
+const null_node: Node.Index = 0;
+
/// Represents in-progress parsing, will be converted to an ast.Tree after completion.
const Parser = struct {
- arena: std.heap.ArenaAllocator,
gpa: *Allocator,
source: []const u8,
- token_ids: []const Token.Id,
- token_locs: []const Token.Loc,
+ token_tags: []const Token.Tag,
+ token_starts: []const ast.ByteOffset,
tok_i: TokenIndex,
errors: std.ArrayListUnmanaged(AstError),
+ nodes: ast.NodeList,
+ extra_data: std.ArrayListUnmanaged(Node.Index),
+
+ const SmallSpan = union(enum) {
+ zero_or_one: Node.Index,
+ multi: []Node.Index,
- /// Root <- skip ContainerMembers eof
- fn parseRoot(p: *Parser) Allocator.Error!*Node.Root {
- const decls = try parseContainerMembers(p, true);
- defer p.gpa.free(decls);
+ fn deinit(self: SmallSpan, gpa: *Allocator) void {
+ switch (self) {
+ .zero_or_one => {},
+ .multi => |list| gpa.free(list),
+ }
+ }
+ };
- // parseContainerMembers will try to skip as much
- // invalid tokens as it can so this can only be the EOF
- const eof_token = p.eatToken(.Eof).?;
+ fn listToSpan(p: *Parser, list: []const Node.Index) !Node.SubRange {
+ try p.extra_data.appendSlice(p.gpa, list);
+ return Node.SubRange{
+ .start = @intCast(Node.Index, p.extra_data.items.len - list.len),
+ .end = @intCast(Node.Index, p.extra_data.items.len),
+ };
+ }
- const decls_len = @intCast(NodeIndex, decls.len);
- const node = try Node.Root.create(&p.arena.allocator, decls_len, eof_token);
- std.mem.copy(*Node, node.decls(), decls);
+ fn addNode(p: *Parser, elem: ast.NodeList.Elem) Allocator.Error!Node.Index {
+ const result = @intCast(Node.Index, p.nodes.len);
+ try p.nodes.append(p.gpa, elem);
+ return result;
+ }
- return node;
+ fn addExtra(p: *Parser, extra: anytype) Allocator.Error!Node.Index {
+ const fields = std.meta.fields(@TypeOf(extra));
+ try p.extra_data.ensureCapacity(p.gpa, p.extra_data.items.len + fields.len);
+ const result = @intCast(u32, p.extra_data.items.len);
+ inline for (fields) |field| {
+ comptime assert(field.field_type == Node.Index);
+ p.extra_data.appendAssumeCapacity(@field(extra, field.name));
+ }
+ return result;
+ }
+
+ fn warn(p: *Parser, msg: ast.Error) error{OutOfMemory}!void {
+ @setCold(true);
+ try p.errors.append(p.gpa, msg);
+ }
+
+ fn fail(p: *Parser, msg: ast.Error) error{ ParseError, OutOfMemory } {
+ @setCold(true);
+ try p.warn(msg);
+ return error.ParseError;
}
/// ContainerMembers
@@ -99,8 +152,8 @@ const Parser = struct {
/// / ContainerField COMMA ContainerMembers
/// / ContainerField
/// /
- fn parseContainerMembers(p: *Parser, top_level: bool) ![]*Node {
- var list = std.ArrayList(*Node).init(p.gpa);
+ fn parseContainerMembers(p: *Parser, top_level: bool) !Node.SubRange {
+ var list = std.ArrayList(Node.Index).init(p.gpa);
defer list.deinit();
var field_state: union(enum) {
@@ -115,103 +168,98 @@ const Parser = struct {
err,
} = .none;
- while (true) {
- if (try p.parseContainerDocComments()) |node| {
- try list.append(node);
- continue;
- }
+ // Skip container doc comments.
+ while (p.eatToken(.ContainerDocComment)) |_| {}
- const doc_comments = try p.parseDocComment();
+ while (true) {
+ const doc_comment = p.eatDocComments();
- if (p.parseTestDecl() catch |err| switch (err) {
+ const test_decl_node = p.parseTestDecl() catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.ParseError => {
p.findNextContainerMember();
continue;
},
- }) |node| {
+ };
+ if (test_decl_node != 0) {
if (field_state == .seen) {
- field_state = .{ .end = node.firstToken() };
+ field_state = .{ .end = p.nodes.items(.main_token)[test_decl_node] };
}
- node.cast(Node.TestDecl).?.doc_comments = doc_comments;
- try list.append(node);
+ try list.append(test_decl_node);
continue;
}
- if (p.parseTopLevelComptime() catch |err| switch (err) {
+ const comptime_node = p.parseTopLevelComptime() catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.ParseError => {
p.findNextContainerMember();
continue;
},
- }) |node| {
+ };
+ if (comptime_node != 0) {
if (field_state == .seen) {
- field_state = .{ .end = node.firstToken() };
+ field_state = .{ .end = p.nodes.items(.main_token)[comptime_node] };
}
- node.cast(Node.Comptime).?.doc_comments = doc_comments;
- try list.append(node);
+ try list.append(comptime_node);
continue;
}
const visib_token = p.eatToken(.Keyword_pub);
- if (p.parseTopLevelDecl(doc_comments, visib_token) catch |err| switch (err) {
+ const top_level_decl = p.parseTopLevelDecl() catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.ParseError => {
p.findNextContainerMember();
continue;
},
- }) |node| {
+ };
+ if (top_level_decl != 0) {
if (field_state == .seen) {
- field_state = .{ .end = visib_token orelse node.firstToken() };
+ field_state = .{
+ .end = visib_token orelse p.nodes.items(.main_token)[top_level_decl],
+ };
}
- try list.append(node);
+ try list.append(top_level_decl);
continue;
}
if (visib_token != null) {
- try p.errors.append(p.gpa, .{
- .ExpectedPubItem = .{ .token = p.tok_i },
- });
+ try p.warn(.{ .ExpectedPubItem = .{ .token = p.tok_i } });
// ignore this pub
continue;
}
- if (p.parseContainerField() catch |err| switch (err) {
+ const container_field = p.parseContainerField() catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.ParseError => {
// attempt to recover
p.findNextContainerMember();
continue;
},
- }) |node| {
+ };
+ if (container_field != 0) {
switch (field_state) {
.none => field_state = .seen,
.err, .seen => {},
.end => |tok| {
- try p.errors.append(p.gpa, .{
- .DeclBetweenFields = .{ .token = tok },
- });
+ try p.warn(.{ .DeclBetweenFields = .{ .token = tok } });
// continue parsing, error will be reported later
field_state = .err;
},
}
-
- const field = node.cast(Node.ContainerField).?;
- field.doc_comments = doc_comments;
- try list.append(node);
+ try list.append(container_field);
const comma = p.eatToken(.Comma) orelse {
// try to continue parsing
const index = p.tok_i;
p.findNextContainerMember();
- const next = p.token_ids[p.tok_i];
+ const next = p.token_tags[p.tok_i];
switch (next) {
.Eof => {
// no invalid tokens were found
if (index == p.tok_i) break;
// Invalid tokens, add error and exit
- try p.errors.append(p.gpa, .{
+ try p.warn(.{
.ExpectedToken = .{ .token = index, .expected_id = .Comma },
});
break;
@@ -219,35 +267,33 @@ const Parser = struct {
else => {
if (next == .RBrace) {
if (!top_level) break;
- _ = p.nextToken();
+ p.tok_i += 1;
}
// add error and continue
- try p.errors.append(p.gpa, .{
+ try p.warn(.{
.ExpectedToken = .{ .token = index, .expected_id = .Comma },
});
continue;
},
}
};
- if (try p.parseAppendedDocComment(comma)) |appended_comment|
- field.doc_comments = appended_comment;
continue;
}
// Dangling doc comment
- if (doc_comments != null) {
- try p.errors.append(p.gpa, .{
- .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
+ if (doc_comment) |tok| {
+ try p.warn(.{
+ .UnattachedDocComment = .{ .token = tok },
});
}
- const next = p.token_ids[p.tok_i];
+ const next = p.token_tags[p.tok_i];
switch (next) {
.Eof => break,
.Keyword_comptime => {
- _ = p.nextToken();
- try p.errors.append(p.gpa, .{
+ p.tok_i += 1;
+ try p.warn(.{
.ExpectedBlockOrField = .{ .token = p.tok_i },
});
},
@@ -255,20 +301,20 @@ const Parser = struct {
const index = p.tok_i;
if (next == .RBrace) {
if (!top_level) break;
- _ = p.nextToken();
+ p.tok_i += 1;
}
// this was likely not supposed to end yet,
// try to find the next declaration
p.findNextContainerMember();
- try p.errors.append(p.gpa, .{
+ try p.warn(.{
.ExpectedContainerMembers = .{ .token = index },
});
},
}
}
- return list.toOwnedSlice();
+ return p.listToSpan(list.items);
}
/// Attempts to find next container member by searching for certain tokens
@@ -276,7 +322,7 @@ const Parser = struct {
var level: u32 = 0;
while (true) {
const tok = p.nextToken();
- switch (p.token_ids[tok]) {
+ switch (p.token_tags[tok]) {
// any of these can start a new top level declaration
.Keyword_test,
.Keyword_comptime,
@@ -293,7 +339,7 @@ const Parser = struct {
.Identifier,
=> {
if (level == 0) {
- p.putBackToken(tok);
+ p.tok_i -= 1;
return;
}
},
@@ -310,13 +356,13 @@ const Parser = struct {
.RBrace => {
if (level == 0) {
// end of container, exit
- p.putBackToken(tok);
+ p.tok_i -= 1;
return;
}
level -= 1;
},
.Eof => {
- p.putBackToken(tok);
+ p.tok_i -= 1;
return;
},
else => {},
@@ -329,11 +375,11 @@ const Parser = struct {
var level: u32 = 0;
while (true) {
const tok = p.nextToken();
- switch (p.token_ids[tok]) {
+ switch (p.token_tags[tok]) {
.LBrace => level += 1,
.RBrace => {
if (level == 0) {
- p.putBackToken(tok);
+ p.tok_i -= 1;
return;
}
level -= 1;
@@ -344,7 +390,7 @@ const Parser = struct {
}
},
.Eof => {
- p.putBackToken(tok);
+ p.tok_i -= 1;
return;
},
else => {},
@@ -352,328 +398,315 @@ const Parser = struct {
}
}
- /// Eat a multiline container doc comment
- fn parseContainerDocComments(p: *Parser) !?*Node {
- if (p.eatToken(.ContainerDocComment)) |first_line| {
- while (p.eatToken(.ContainerDocComment)) |_| {}
- const node = try p.arena.allocator.create(Node.DocComment);
- node.* = .{ .first_line = first_line };
- return &node.base;
- }
- return null;
- }
-
- /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
- fn parseTestDecl(p: *Parser) !?*Node {
- const test_token = p.eatToken(.Keyword_test) orelse return null;
- const name_node = try p.parseStringLiteralSingle();
- const block_node = (try p.parseBlock(null)) orelse {
- try p.errors.append(p.gpa, .{ .ExpectedLBrace = .{ .token = p.tok_i } });
- return error.ParseError;
- };
-
- const test_node = try p.arena.allocator.create(Node.TestDecl);
- test_node.* = .{
- .doc_comments = null,
- .test_token = test_token,
- .name = name_node,
- .body_node = block_node,
- };
- return &test_node.base;
+ /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE? Block
+ fn parseTestDecl(p: *Parser) !Node.Index {
+ const test_token = p.eatToken(.Keyword_test) orelse return null_node;
+ const name_token = try p.expectToken(.StringLiteral);
+ const block_node = try p.parseBlock();
+ if (block_node == 0) return p.fail(.{ .ExpectedLBrace = .{ .token = p.tok_i } });
+ return p.addNode(.{
+ .tag = .TestDecl,
+ .main_token = test_token,
+ .data = .{
+ .lhs = name_token,
+ .rhs = block_node,
+ },
+ });
}
/// TopLevelComptime <- KEYWORD_comptime BlockExpr
- fn parseTopLevelComptime(p: *Parser) !?*Node {
- const tok = p.eatToken(.Keyword_comptime) orelse return null;
- const lbrace = p.eatToken(.LBrace) orelse {
- p.putBackToken(tok);
- return null;
- };
- p.putBackToken(lbrace);
- const block_node = try p.expectNode(parseBlockExpr, .{
- .ExpectedLabelOrLBrace = .{ .token = p.tok_i },
- });
-
- const comptime_node = try p.arena.allocator.create(Node.Comptime);
- comptime_node.* = .{
- .doc_comments = null,
- .comptime_token = tok,
- .expr = block_node,
- };
- return &comptime_node.base;
+ fn parseTopLevelComptime(p: *Parser) !Node.Index {
+ if (p.token_tags[p.tok_i] == .Keyword_comptime and
+ p.token_tags[p.tok_i + 1] == .LBrace)
+ {
+ return p.addNode(.{
+ .tag = .Comptime,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.parseBlock(),
+ .rhs = undefined,
+ },
+ });
+ } else {
+ return null_node;
+ }
}
/// TopLevelDecl
/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
/// / KEYWORD_usingnamespace Expr SEMICOLON
- fn parseTopLevelDecl(p: *Parser, doc_comments: ?*Node.DocComment, visib_token: ?TokenIndex) !?*Node {
- var lib_name: ?*Node = null;
- const extern_export_inline_token = blk: {
- if (p.eatToken(.Keyword_export)) |token| break :blk token;
- if (p.eatToken(.Keyword_extern)) |token| {
- lib_name = try p.parseStringLiteralSingle();
- break :blk token;
+ fn parseTopLevelDecl(p: *Parser) !Node.Index {
+ const extern_export_inline_token = p.nextToken();
+ var expect_fn: bool = false;
+ var exported: bool = false;
+ switch (p.token_tags[extern_export_inline_token]) {
+ .Keyword_extern => _ = p.eatToken(.StringLiteral),
+ .Keyword_export => exported = true,
+ .Keyword_inline, .Keyword_noinline => expect_fn = true,
+ else => p.tok_i -= 1,
+ }
+ const fn_proto = try p.parseFnProto();
+ if (fn_proto != 0) {
+ switch (p.token_tags[p.tok_i]) {
+ .Semicolon => {
+ const semicolon_token = p.nextToken();
+ try p.parseAppendedDocComment(semicolon_token);
+ return fn_proto;
+ },
+ .LBrace => {
+ const body_block = try p.parseBlock();
+ assert(body_block != 0);
+ return p.addNode(.{
+ .tag = .FnDecl,
+ .main_token = p.nodes.items(.main_token)[fn_proto],
+ .data = .{
+ .lhs = fn_proto,
+ .rhs = body_block,
+ },
+ });
+ },
+ else => {
+ // Since parseBlock only return error.ParseError on
+ // a missing '}' we can assume this function was
+ // supposed to end here.
+ try p.warn(.{ .ExpectedSemiOrLBrace = .{ .token = p.tok_i } });
+ return null_node;
+ },
}
- if (p.eatToken(.Keyword_inline)) |token| break :blk token;
- if (p.eatToken(.Keyword_noinline)) |token| break :blk token;
- break :blk null;
- };
-
- if (try p.parseFnProto(.top_level, .{
- .doc_comments = doc_comments,
- .visib_token = visib_token,
- .extern_export_inline_token = extern_export_inline_token,
- .lib_name = lib_name,
- })) |node| {
- return node;
}
-
- if (extern_export_inline_token) |token| {
- if (p.token_ids[token] == .Keyword_inline or
- p.token_ids[token] == .Keyword_noinline)
- {
- try p.errors.append(p.gpa, .{
- .ExpectedFn = .{ .token = p.tok_i },
- });
- return error.ParseError;
- }
+ if (expect_fn) {
+ try p.warn(.{
+ .ExpectedFn = .{ .token = p.tok_i },
+ });
+ return error.ParseError;
}
const thread_local_token = p.eatToken(.Keyword_threadlocal);
-
- if (try p.parseVarDecl(.{
- .doc_comments = doc_comments,
- .visib_token = visib_token,
- .thread_local_token = thread_local_token,
- .extern_export_token = extern_export_inline_token,
- .lib_name = lib_name,
- })) |node| {
- return node;
+ const var_decl = try p.parseVarDecl();
+ if (var_decl != 0) {
+ const semicolon_token = try p.expectToken(.Semicolon);
+ try p.parseAppendedDocComment(semicolon_token);
+ return var_decl;
}
-
if (thread_local_token != null) {
- try p.errors.append(p.gpa, .{
- .ExpectedVarDecl = .{ .token = p.tok_i },
- });
- // ignore this and try again;
- return error.ParseError;
+ return p.fail(.{ .ExpectedVarDecl = .{ .token = p.tok_i } });
}
- if (extern_export_inline_token) |token| {
- try p.errors.append(p.gpa, .{
- .ExpectedVarDeclOrFn = .{ .token = p.tok_i },
- });
- // ignore this and try again;
- return error.ParseError;
+ if (exported) {
+ return p.fail(.{ .ExpectedVarDeclOrFn = .{ .token = p.tok_i } });
}
- const use_token = p.eatToken(.Keyword_usingnamespace) orelse return null;
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ const usingnamespace_token = p.eatToken(.Keyword_usingnamespace) orelse return null_node;
+ const expr = try p.expectExpr();
const semicolon_token = try p.expectToken(.Semicolon);
-
- const node = try p.arena.allocator.create(Node.Use);
- node.* = .{
- .doc_comments = doc_comments orelse try p.parseAppendedDocComment(semicolon_token),
- .visib_token = visib_token,
- .use_token = use_token,
- .expr = expr,
- .semicolon_token = semicolon_token,
- };
-
- return &node.base;
+ try p.parseAppendedDocComment(semicolon_token);
+ return p.addNode(.{
+ .tag = .UsingNamespace,
+ .main_token = usingnamespace_token,
+ .data = .{
+ .lhs = expr,
+ .rhs = undefined,
+ },
+ });
}
/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? CallConv? EXCLAMATIONMARK? (Keyword_anytype / TypeExpr)
- fn parseFnProto(p: *Parser, level: enum { top_level, as_type }, fields: struct {
- doc_comments: ?*Node.DocComment = null,
- visib_token: ?TokenIndex = null,
- extern_export_inline_token: ?TokenIndex = null,
- lib_name: ?*Node = null,
- }) !?*Node {
- // TODO: Remove once extern/async fn rewriting is
- var is_async: ?void = null;
- var is_extern_prototype: ?void = null;
- const cc_token: ?TokenIndex = blk: {
- if (p.eatToken(.Keyword_extern)) |token| {
- is_extern_prototype = {};
- break :blk token;
- }
- if (p.eatToken(.Keyword_async)) |token| {
- is_async = {};
- break :blk token;
- }
- break :blk null;
- };
- const fn_token = p.eatToken(.Keyword_fn) orelse {
- if (cc_token) |token|
- p.putBackToken(token);
- return null;
- };
- const name_token = p.eatToken(.Identifier);
- const lparen = try p.expectToken(.LParen);
+ fn parseFnProto(p: *Parser) !Node.Index {
+ const fn_token = p.eatToken(.Keyword_fn) orelse return null_node;
+ _ = p.eatToken(.Identifier);
const params = try p.parseParamDeclList();
- defer p.gpa.free(params);
- const var_args_token = p.eatToken(.Ellipsis3);
- const rparen = try p.expectToken(.RParen);
+ defer params.deinit(p.gpa);
const align_expr = try p.parseByteAlign();
const section_expr = try p.parseLinkSection();
const callconv_expr = try p.parseCallconv();
- const exclamation_token = p.eatToken(.Bang);
+ const bang_token = p.eatToken(.Bang);
- const return_type_expr = (try p.parseAnyType()) orelse
- try p.expectNodeRecoverable(parseTypeExpr, .{
+ const return_type_expr = try p.parseTypeExpr();
+ if (return_type_expr == 0) {
// most likely the user forgot to specify the return type.
// Mark return type as invalid and try to continue.
- .ExpectedReturnType = .{ .token = p.tok_i },
- });
-
- // TODO https://github.com/ziglang/zig/issues/3750
- const R = Node.FnProto.ReturnType;
- const return_type = if (return_type_expr == null)
- R{ .Invalid = rparen }
- else if (exclamation_token != null)
- R{ .InferErrorSet = return_type_expr.? }
- else
- R{ .Explicit = return_type_expr.? };
+ try p.warn(.{ .ExpectedReturnType = .{ .token = p.tok_i } });
+ }
- const body_node: ?*Node = switch (level) {
- .top_level => blk: {
- if (p.eatToken(.Semicolon)) |_| {
- break :blk null;
- }
- const body_block = (try p.parseBlock(null)) orelse {
- // Since parseBlock only return error.ParseError on
- // a missing '}' we can assume this function was
- // supposed to end here.
- try p.errors.append(p.gpa, .{ .ExpectedSemiOrLBrace = .{ .token = p.tok_i } });
- break :blk null;
- };
- break :blk body_block;
+ if (align_expr == 0 and section_expr == 0 and callconv_expr == 0) {
+ switch (params) {
+ .zero_or_one => |param| return p.addNode(.{
+ .tag = .FnProtoSimple,
+ .main_token = fn_token,
+ .data = .{
+ .lhs = param,
+ .rhs = return_type_expr,
+ },
+ }),
+ .multi => |list| {
+ const span = try p.listToSpan(list);
+ return p.addNode(.{
+ .tag = .FnProtoSimpleMulti,
+ .main_token = fn_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.SubRange{
+ .start = span.start,
+ .end = span.end,
+ }),
+ .rhs = return_type_expr,
+ },
+ });
+ },
+ }
+ }
+ switch (params) {
+ .zero_or_one => |param| return p.addNode(.{
+ .tag = .FnProtoOne,
+ .main_token = fn_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.FnProtoOne{
+ .param = param,
+ .align_expr = align_expr,
+ .section_expr = section_expr,
+ .callconv_expr = callconv_expr,
+ }),
+ .rhs = return_type_expr,
+ },
+ }),
+ .multi => |list| {
+ const span = try p.listToSpan(list);
+ return p.addNode(.{
+ .tag = .FnProto,
+ .main_token = fn_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.FnProto{
+ .params_start = span.start,
+ .params_end = span.end,
+ .align_expr = align_expr,
+ .section_expr = section_expr,
+ .callconv_expr = callconv_expr,
+ }),
+ .rhs = return_type_expr,
+ },
+ });
},
- .as_type => null,
- };
-
- const fn_proto_node = try Node.FnProto.create(&p.arena.allocator, .{
- .params_len = params.len,
- .fn_token = fn_token,
- .return_type = return_type,
- }, .{
- .doc_comments = fields.doc_comments,
- .visib_token = fields.visib_token,
- .name_token = name_token,
- .var_args_token = var_args_token,
- .extern_export_inline_token = fields.extern_export_inline_token,
- .body_node = body_node,
- .lib_name = fields.lib_name,
- .align_expr = align_expr,
- .section_expr = section_expr,
- .callconv_expr = callconv_expr,
- .is_extern_prototype = is_extern_prototype,
- .is_async = is_async,
- });
- std.mem.copy(Node.FnProto.ParamDecl, fn_proto_node.params(), params);
-
- return &fn_proto_node.base;
+ }
}
/// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON
- fn parseVarDecl(p: *Parser, fields: struct {
- doc_comments: ?*Node.DocComment = null,
- visib_token: ?TokenIndex = null,
- thread_local_token: ?TokenIndex = null,
- extern_export_token: ?TokenIndex = null,
- lib_name: ?*Node = null,
- comptime_token: ?TokenIndex = null,
- }) !?*Node {
+ fn parseVarDecl(p: *Parser) !Node.Index {
const mut_token = p.eatToken(.Keyword_const) orelse
p.eatToken(.Keyword_var) orelse
- return null;
+ return null_node;
const name_token = try p.expectToken(.Identifier);
- const type_node = if (p.eatToken(.Colon) != null)
- try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
- })
- else
- null;
+ const type_node: Node.Index = if (p.eatToken(.Colon) == null) 0 else try p.expectTypeExpr();
const align_node = try p.parseByteAlign();
const section_node = try p.parseLinkSection();
- const eq_token = p.eatToken(.Equal);
- const init_node = if (eq_token != null) blk: {
- break :blk try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
+ const init_node: Node.Index = if (p.eatToken(.Equal) == null) 0 else try p.expectExpr();
+ if (section_node == 0) {
+ if (align_node == 0) {
+ return p.addNode(.{
+ .tag = .SimpleVarDecl,
+ .main_token = mut_token,
+ .data = .{
+ .lhs = type_node,
+ .rhs = init_node,
+ },
+ });
+ } else if (type_node == 0) {
+ return p.addNode(.{
+ .tag = .AlignedVarDecl,
+ .main_token = mut_token,
+ .data = .{
+ .lhs = align_node,
+ .rhs = init_node,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .LocalVarDecl,
+ .main_token = mut_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.LocalVarDecl{
+ .type_node = type_node,
+ .align_node = align_node,
+ }),
+ .rhs = init_node,
+ },
+ });
+ }
+ } else {
+ return p.addNode(.{
+ .tag = .GlobalVarDecl,
+ .main_token = mut_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.GlobalVarDecl{
+ .type_node = type_node,
+ .align_node = align_node,
+ .section_node = section_node,
+ }),
+ .rhs = init_node,
+ },
});
- } else null;
- const semicolon_token = try p.expectToken(.Semicolon);
-
- const doc_comments = fields.doc_comments orelse try p.parseAppendedDocComment(semicolon_token);
-
- const node = try Node.VarDecl.create(&p.arena.allocator, .{
- .mut_token = mut_token,
- .name_token = name_token,
- .semicolon_token = semicolon_token,
- }, .{
- .doc_comments = doc_comments,
- .visib_token = fields.visib_token,
- .thread_local_token = fields.thread_local_token,
- .eq_token = eq_token,
- .comptime_token = fields.comptime_token,
- .extern_export_token = fields.extern_export_token,
- .lib_name = fields.lib_name,
- .type_node = type_node,
- .align_node = align_node,
- .section_node = section_node,
- .init_node = init_node,
- });
- return &node.base;
+ }
}
/// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
- fn parseContainerField(p: *Parser) !?*Node {
+ fn parseContainerField(p: *Parser) !Node.Index {
const comptime_token = p.eatToken(.Keyword_comptime);
const name_token = p.eatToken(.Identifier) orelse {
- if (comptime_token) |t| p.putBackToken(t);
- return null;
+ if (comptime_token) |_| p.tok_i -= 1;
+ return null_node;
};
- var align_expr: ?*Node = null;
- var type_expr: ?*Node = null;
+ var align_expr: Node.Index = 0;
+ var type_expr: Node.Index = 0;
if (p.eatToken(.Colon)) |_| {
- if (p.eatToken(.Keyword_anytype) orelse p.eatToken(.Keyword_var)) |anytype_tok| {
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .AnyType },
- .token = anytype_tok,
- };
- type_expr = &node.base;
- } else {
- type_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
+ if (p.eatToken(.Keyword_anytype)) |anytype_tok| {
+ type_expr = try p.addNode(.{
+ .tag = .AnyType,
+ .main_token = anytype_tok,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
});
+ } else {
+ type_expr = try p.expectTypeExpr();
align_expr = try p.parseByteAlign();
}
}
- const value_expr = if (p.eatToken(.Equal)) |_|
- try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- })
- else
- null;
-
- const node = try p.arena.allocator.create(Node.ContainerField);
- node.* = .{
- .doc_comments = null,
- .comptime_token = comptime_token,
- .name_token = name_token,
- .type_expr = type_expr,
- .value_expr = value_expr,
- .align_expr = align_expr,
- };
- return &node.base;
+ const value_expr: Node.Index = if (p.eatToken(.Equal) == null) 0 else try p.expectExpr();
+
+ if (align_expr == 0) {
+ return p.addNode(.{
+ .tag = .ContainerFieldInit,
+ .main_token = name_token,
+ .data = .{
+ .lhs = type_expr,
+ .rhs = value_expr,
+ },
+ });
+ } else if (value_expr == 0) {
+ return p.addNode(.{
+ .tag = .ContainerFieldAlign,
+ .main_token = name_token,
+ .data = .{
+ .lhs = type_expr,
+ .rhs = align_expr,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .ContainerField,
+ .main_token = name_token,
+ .data = .{
+ .lhs = type_expr,
+ .rhs = try p.addExtra(Node.ContainerField{
+ .value_expr = value_expr,
+ .align_expr = align_expr,
+ }),
+ },
+ });
+ }
}
/// Statement
@@ -687,833 +720,1475 @@ const Parser = struct {
/// / LabeledStatement
/// / SwitchExpr
/// / AssignExpr SEMICOLON
- fn parseStatement(p: *Parser) Error!?*Node {
+ fn parseStatement(p: *Parser) Error!Node.Index {
const comptime_token = p.eatToken(.Keyword_comptime);
- if (try p.parseVarDecl(.{
- .comptime_token = comptime_token,
- })) |node| {
- return node;
+ const var_decl = try p.parseVarDecl();
+ if (var_decl != 0) {
+ _ = try p.expectTokenRecoverable(.Semicolon);
+ return var_decl;
}
if (comptime_token) |token| {
- const block_expr = try p.expectNode(parseBlockExprStatement, .{
- .ExpectedBlockOrAssignment = .{ .token = p.tok_i },
+ return p.addNode(.{
+ .tag = .Comptime,
+ .main_token = token,
+ .data = .{
+ .lhs = try p.expectBlockExprStatement(),
+ .rhs = undefined,
+ },
});
-
- const node = try p.arena.allocator.create(Node.Comptime);
- node.* = .{
- .doc_comments = null,
- .comptime_token = token,
- .expr = block_expr,
- };
- return &node.base;
}
- if (p.eatToken(.Keyword_nosuspend)) |nosuspend_token| {
- const block_expr = try p.expectNode(parseBlockExprStatement, .{
- .ExpectedBlockOrAssignment = .{ .token = p.tok_i },
- });
-
- const node = try p.arena.allocator.create(Node.Nosuspend);
- node.* = .{
- .nosuspend_token = nosuspend_token,
- .expr = block_expr,
- };
- return &node.base;
+ const token = p.nextToken();
+ switch (p.token_tags[token]) {
+ .Keyword_nosuspend => {
+ return p.addNode(.{
+ .tag = .Nosuspend,
+ .main_token = token,
+ .data = .{
+ .lhs = try p.expectBlockExprStatement(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_suspend => {
+ const block_expr: Node.Index = if (p.eatToken(.Semicolon) != null)
+ 0
+ else
+ try p.expectBlockExprStatement();
+ return p.addNode(.{
+ .tag = .Suspend,
+ .main_token = token,
+ .data = .{
+ .lhs = block_expr,
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_defer => return p.addNode(.{
+ .tag = .Defer,
+ .main_token = token,
+ .data = .{
+ .lhs = undefined,
+ .rhs = try p.expectBlockExprStatement(),
+ },
+ }),
+ .Keyword_errdefer => return p.addNode(.{
+ .tag = .ErrDefer,
+ .main_token = token,
+ .data = .{
+ .lhs = try p.parsePayload(),
+ .rhs = try p.expectBlockExprStatement(),
+ },
+ }),
+ else => p.tok_i -= 1,
}
- if (p.eatToken(.Keyword_suspend)) |suspend_token| {
- const semicolon = p.eatToken(.Semicolon);
-
- const body_node = if (semicolon == null) blk: {
- break :blk try p.expectNode(parseBlockExprStatement, .{
- .ExpectedBlockOrExpression = .{ .token = p.tok_i },
- });
- } else null;
+ const if_statement = try p.parseIfStatement();
+ if (if_statement != 0) return if_statement;
- const node = try p.arena.allocator.create(Node.Suspend);
- node.* = .{
- .suspend_token = suspend_token,
- .body = body_node,
- };
- return &node.base;
- }
+ const labeled_statement = try p.parseLabeledStatement();
+ if (labeled_statement != 0) return labeled_statement;
- const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);
- if (defer_token) |token| {
- const payload = if (p.token_ids[token] == .Keyword_errdefer)
- try p.parsePayload()
- else
- null;
- const expr_node = try p.expectNode(parseBlockExprStatement, .{
- .ExpectedBlockOrExpression = .{ .token = p.tok_i },
- });
- const node = try p.arena.allocator.create(Node.Defer);
- node.* = .{
- .defer_token = token,
- .expr = expr_node,
- .payload = payload,
- };
- return &node.base;
- }
+ const switch_expr = try p.parseSwitchExpr();
+ if (switch_expr != 0) return switch_expr;
- if (try p.parseIfStatement()) |node| return node;
- if (try p.parseLabeledStatement()) |node| return node;
- if (try p.parseSwitchExpr()) |node| return node;
- if (try p.parseAssignExpr()) |node| {
+ const assign_expr = try p.parseAssignExpr();
+ if (assign_expr != 0) {
_ = try p.expectTokenRecoverable(.Semicolon);
- return node;
+ return assign_expr;
}
- return null;
+ return null_node;
+ }
+
+ fn expectStatement(p: *Parser) !Node.Index {
+ const statement = try p.parseStatement();
+ if (statement == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ }
+ return statement;
}
/// IfStatement
/// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )?
/// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
- fn parseIfStatement(p: *Parser) !?*Node {
- const if_node = (try p.parseIfPrefix()) orelse return null;
- const if_prefix = if_node.cast(Node.If).?;
-
- const block_expr = (try p.parseBlockExpr());
- const assign_expr = if (block_expr == null)
- try p.expectNode(parseAssignExpr, .{
- .ExpectedBlockOrAssignment = .{ .token = p.tok_i },
- })
- else
- null;
-
- const semicolon = if (assign_expr != null) p.eatToken(.Semicolon) else null;
-
- const else_node = if (semicolon == null) blk: {
- const else_token = p.eatToken(.Keyword_else) orelse break :blk null;
- const payload = try p.parsePayload();
- const else_body = try p.expectNode(parseStatement, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
-
- const node = try p.arena.allocator.create(Node.Else);
- node.* = .{
- .else_token = else_token,
- .payload = payload,
- .body = else_body,
- };
-
- break :blk node;
- } else null;
-
- if (block_expr) |body| {
- if_prefix.body = body;
- if_prefix.@"else" = else_node;
- return if_node;
- }
-
- if (assign_expr) |body| {
- if_prefix.body = body;
- if (semicolon != null) return if_node;
- if (else_node != null) {
- if_prefix.@"else" = else_node;
- return if_node;
+ fn parseIfStatement(p: *Parser) !Node.Index {
+ const if_token = p.eatToken(.Keyword_if) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const condition = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ const then_payload = try p.parsePtrPayload();
+
+ // TODO propose to change the syntax so that semicolons are always required
+ // inside if statements, even if there is an `else`.
+ var else_required = false;
+ const then_expr = blk: {
+ const block_expr = try p.parseBlockExpr();
+ if (block_expr != 0) break :blk block_expr;
+ const assign_expr = try p.parseAssignExpr();
+ if (assign_expr == 0) {
+ return p.fail(.{ .ExpectedBlockOrAssignment = .{ .token = p.tok_i } });
+ }
+ if (p.eatToken(.Semicolon)) |_| {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .IfSimple else .IfSimpleOptional,
+ .main_token = if_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = assign_expr,
+ },
+ });
+ }
+ else_required = true;
+ break :blk assign_expr;
+ };
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ if (else_required) {
+ return p.fail(.{ .ExpectedSemiOrElse = .{ .token = p.tok_i } });
}
- try p.errors.append(p.gpa, .{
- .ExpectedSemiOrElse = .{ .token = p.tok_i },
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .IfSimple else .IfSimpleOptional,
+ .main_token = if_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = then_expr,
+ },
});
- }
-
- return if_node;
+ };
+ const else_payload = try p.parsePayload();
+ const else_expr = try p.expectStatement();
+ const tag = if (else_payload != 0)
+ Node.Tag.IfError
+ else if (then_payload != 0)
+ Node.Tag.IfOptional
+ else
+ Node.Tag.If;
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = if_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.If{
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
+ });
}
/// LabeledStatement <- BlockLabel? (Block / LoopStatement)
- fn parseLabeledStatement(p: *Parser) !?*Node {
- var colon: TokenIndex = undefined;
- const label_token = p.parseBlockLabel(&colon);
-
- if (try p.parseBlock(label_token)) |node| return node;
-
- if (try p.parseLoopStatement()) |node| {
- if (node.cast(Node.For)) |for_node| {
- for_node.label = label_token;
- } else if (node.cast(Node.While)) |while_node| {
- while_node.label = label_token;
- } else unreachable;
- return node;
- }
+ fn parseLabeledStatement(p: *Parser) !Node.Index {
+ const label_token = p.parseBlockLabel();
+ const block = try p.parseBlock();
+ if (block != 0) return block;
- if (label_token != null) {
- try p.errors.append(p.gpa, .{
- .ExpectedLabelable = .{ .token = p.tok_i },
- });
- return error.ParseError;
+ const loop_stmt = try p.parseLoopStatement();
+ if (loop_stmt != 0) return loop_stmt;
+
+ if (label_token != 0) {
+ return p.fail(.{ .ExpectedLabelable = .{ .token = p.tok_i } });
}
- return null;
+ return null_node;
}
/// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement)
- fn parseLoopStatement(p: *Parser) !?*Node {
+ fn parseLoopStatement(p: *Parser) !Node.Index {
const inline_token = p.eatToken(.Keyword_inline);
- if (try p.parseForStatement()) |node| {
- node.cast(Node.For).?.inline_token = inline_token;
- return node;
- }
+ const for_statement = try p.parseForStatement();
+ if (for_statement != 0) return for_statement;
- if (try p.parseWhileStatement()) |node| {
- node.cast(Node.While).?.inline_token = inline_token;
- return node;
- }
- if (inline_token == null) return null;
+ const while_statement = try p.parseWhileStatement();
+ if (while_statement != 0) return while_statement;
+
+ if (inline_token == null) return null_node;
// If we've seen "inline", there should have been a "for" or "while"
- try p.errors.append(p.gpa, .{
- .ExpectedInlinable = .{ .token = p.tok_i },
- });
- return error.ParseError;
+ return p.fail(.{ .ExpectedInlinable = .{ .token = p.tok_i } });
}
+ /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
/// ForStatement
/// <- ForPrefix BlockExpr ( KEYWORD_else Statement )?
/// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement )
- fn parseForStatement(p: *Parser) !?*Node {
- const node = (try p.parseForPrefix()) orelse return null;
- const for_prefix = node.cast(Node.For).?;
-
- if (try p.parseBlockExpr()) |block_expr_node| {
- for_prefix.body = block_expr_node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const statement_node = try p.expectNode(parseStatement, .{
- .InvalidToken = .{ .token = p.tok_i },
+ fn parseForStatement(p: *Parser) !Node.Index {
+ const for_token = p.eatToken(.Keyword_for) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const array_expr = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ _ = try p.parsePtrIndexPayload();
+
+ // TODO propose to change the syntax so that semicolons are always required
+ // inside while statements, even if there is an `else`.
+ var else_required = false;
+ const then_expr = blk: {
+ const block_expr = try p.parseBlockExpr();
+ if (block_expr != 0) break :blk block_expr;
+ const assign_expr = try p.parseAssignExpr();
+ if (assign_expr == 0) {
+ return p.fail(.{ .ExpectedBlockOrAssignment = .{ .token = p.tok_i } });
+ }
+ if (p.eatToken(.Semicolon)) |_| {
+ return p.addNode(.{
+ .tag = .ForSimple,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = assign_expr,
+ },
});
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = null,
- .body = statement_node,
- };
- for_prefix.@"else" = else_node;
-
- return node;
}
-
- return node;
- }
-
- for_prefix.body = try p.expectNode(parseAssignExpr, .{
- .ExpectedBlockOrAssignment = .{ .token = p.tok_i },
- });
-
- if (p.eatToken(.Semicolon) != null) return node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const statement_node = try p.expectNode(parseStatement, .{
- .ExpectedStatement = .{ .token = p.tok_i },
+ else_required = true;
+ break :blk assign_expr;
+ };
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ if (else_required) {
+ return p.fail(.{ .ExpectedSemiOrElse = .{ .token = p.tok_i } });
+ }
+ return p.addNode(.{
+ .tag = .ForSimple,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = then_expr,
+ },
});
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = null,
- .body = statement_node,
- };
- for_prefix.@"else" = else_node;
- return node;
- }
-
- try p.errors.append(p.gpa, .{
- .ExpectedSemiOrElse = .{ .token = p.tok_i },
+ };
+ return p.addNode(.{
+ .tag = .For,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = try p.addExtra(Node.If{
+ .then_expr = then_expr,
+ .else_expr = try p.expectStatement(),
+ }),
+ },
});
-
- return node;
}
+ /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
/// WhileStatement
/// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )?
/// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement )
- fn parseWhileStatement(p: *Parser) !?*Node {
- const node = (try p.parseWhilePrefix()) orelse return null;
- const while_prefix = node.cast(Node.While).?;
-
- if (try p.parseBlockExpr()) |block_expr_node| {
- while_prefix.body = block_expr_node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const payload = try p.parsePayload();
-
- const statement_node = try p.expectNode(parseStatement, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = payload,
- .body = statement_node,
- };
- while_prefix.@"else" = else_node;
+ fn parseWhileStatement(p: *Parser) !Node.Index {
+ const while_token = p.eatToken(.Keyword_while) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const condition = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ const then_payload = try p.parsePtrPayload();
+ const continue_expr = try p.parseWhileContinueExpr();
- return node;
+ // TODO propose to change the syntax so that semicolons are always required
+ // inside while statements, even if there is an `else`.
+ var else_required = false;
+ const then_expr = blk: {
+ const block_expr = try p.parseBlockExpr();
+ if (block_expr != 0) break :blk block_expr;
+ const assign_expr = try p.parseAssignExpr();
+ if (assign_expr == 0) {
+ return p.fail(.{ .ExpectedBlockOrAssignment = .{ .token = p.tok_i } });
}
-
- return node;
- }
-
- while_prefix.body = try p.expectNode(parseAssignExpr, .{
- .ExpectedBlockOrAssignment = .{ .token = p.tok_i },
- });
-
- if (p.eatToken(.Semicolon) != null) return node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const payload = try p.parsePayload();
-
- const statement_node = try p.expectNode(parseStatement, .{
- .ExpectedStatement = .{ .token = p.tok_i },
- });
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = payload,
- .body = statement_node,
- };
- while_prefix.@"else" = else_node;
- return node;
- }
-
- try p.errors.append(p.gpa, .{
- .ExpectedSemiOrElse = .{ .token = p.tok_i },
+ if (p.eatToken(.Semicolon)) |_| {
+ if (continue_expr == 0) {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileSimple else .WhileSimpleOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = assign_expr,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileCont else .WhileContOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.WhileCont{
+ .continue_expr = continue_expr,
+ .then_expr = assign_expr,
+ }),
+ },
+ });
+ }
+ }
+ else_required = true;
+ break :blk assign_expr;
+ };
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ if (else_required) {
+ return p.fail(.{ .ExpectedSemiOrElse = .{ .token = p.tok_i } });
+ }
+ if (continue_expr == 0) {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileSimple else .WhileSimpleOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = then_expr,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileCont else .WhileContOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.WhileCont{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ }),
+ },
+ });
+ }
+ };
+ const else_payload = try p.parsePayload();
+ const else_expr = try p.expectStatement();
+ const tag = if (else_payload != 0)
+ Node.Tag.WhileError
+ else if (then_payload != 0)
+ Node.Tag.WhileOptional
+ else
+ Node.Tag.While;
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.While{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
});
-
- return node;
}
/// BlockExprStatement
/// <- BlockExpr
/// / AssignExpr SEMICOLON
- fn parseBlockExprStatement(p: *Parser) !?*Node {
- if (try p.parseBlockExpr()) |node| return node;
- if (try p.parseAssignExpr()) |node| {
+ fn parseBlockExprStatement(p: *Parser) !Node.Index {
+ const block_expr = try p.parseBlockExpr();
+ if (block_expr != 0) {
+ return block_expr;
+ }
+ const assign_expr = try p.parseAssignExpr();
+ if (assign_expr != 0) {
_ = try p.expectTokenRecoverable(.Semicolon);
- return node;
+ return assign_expr;
}
- return null;
+ return null_node;
+ }
+
+ fn expectBlockExprStatement(p: *Parser) !Node.Index {
+ const node = try p.parseBlockExprStatement();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedBlockOrExpression = .{ .token = p.tok_i } });
+ }
+ return node;
}
/// BlockExpr <- BlockLabel? Block
- fn parseBlockExpr(p: *Parser) Error!?*Node {
- var colon: TokenIndex = undefined;
- const label_token = p.parseBlockLabel(&colon);
- const block_node = (try p.parseBlock(label_token)) orelse {
- if (label_token) |label| {
- p.putBackToken(label + 1); // ":"
- p.putBackToken(label); // IDENTIFIER
- }
- return null;
- };
- return block_node;
+ fn parseBlockExpr(p: *Parser) Error!Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .Identifier => {
+ if (p.token_tags[p.tok_i + 1] == .Colon and
+ p.token_tags[p.tok_i + 2] == .LBrace)
+ {
+ p.tok_i += 2;
+ return p.parseBlock();
+ } else {
+ return null_node;
+ }
+ },
+ .LBrace => return p.parseBlock(),
+ else => return null_node,
+ }
}
/// AssignExpr <- Expr (AssignOp Expr)?
- fn parseAssignExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseAssignOp, parseExpr, .Once);
- }
+ /// AssignOp
+ /// <- ASTERISKEQUAL
+ /// / SLASHEQUAL
+ /// / PERCENTEQUAL
+ /// / PLUSEQUAL
+ /// / MINUSEQUAL
+ /// / LARROW2EQUAL
+ /// / RARROW2EQUAL
+ /// / AMPERSANDEQUAL
+ /// / CARETEQUAL
+ /// / PIPEEQUAL
+ /// / ASTERISKPERCENTEQUAL
+ /// / PLUSPERCENTEQUAL
+ /// / MINUSPERCENTEQUAL
+ /// / EQUAL
+ fn parseAssignExpr(p: *Parser) !Node.Index {
+ const expr = try p.parseExpr();
+ if (expr == 0) return null_node;
- /// Expr <- BoolOrExpr
- fn parseExpr(p: *Parser) Error!?*Node {
- return p.parsePrefixOpExpr(parseTry, parseBoolOrExpr);
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .AsteriskEqual => .AssignMul,
+ .SlashEqual => .AssignDiv,
+ .PercentEqual => .AssignMod,
+ .PlusEqual => .AssignAdd,
+ .MinusEqual => .AssignSub,
+ .AngleBracketAngleBracketLeftEqual => .AssignBitShiftLeft,
+ .AngleBracketAngleBracketRightEqual => .AssignBitShiftRight,
+ .AmpersandEqual => .AssignBitAnd,
+ .CaretEqual => .AssignBitXor,
+ .PipeEqual => .AssignBitOr,
+ .AsteriskPercentEqual => .AssignMulWrap,
+ .PlusPercentEqual => .AssignAddWrap,
+ .MinusPercentEqual => .AssignSubWrap,
+ .Equal => .Assign,
+ else => return expr,
+ };
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = expr,
+ .rhs = try p.expectExpr(),
+ },
+ });
}
- /// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)*
- fn parseBoolOrExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(
- SimpleBinOpParseFn(.Keyword_or, .BoolOr),
- parseBoolAndExpr,
- .Infinitely,
- );
+ fn expectAssignExpr(p: *Parser) !Node.Index {
+ const expr = try p.parseAssignExpr();
+ if (expr == 0) {
+ return p.fail(.{ .ExpectedExprOrAssignment = .{ .token = p.tok_i } });
+ }
+ return expr;
}
- /// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)*
- fn parseBoolAndExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(
- SimpleBinOpParseFn(.Keyword_and, .BoolAnd),
- parseCompareExpr,
- .Infinitely,
- );
+ /// Expr <- BoolOrExpr
+ fn parseExpr(p: *Parser) Error!Node.Index {
+ return p.parseBoolOrExpr();
}
- /// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)?
- fn parseCompareExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseCompareOp, parseBitwiseExpr, .Once);
+ fn expectExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseExpr();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedExpr = .{ .token = p.tok_i } });
+ } else {
+ return node;
+ }
}
- /// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)*
- fn parseBitwiseExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseBitwiseOp, parseBitShiftExpr, .Infinitely);
- }
+ /// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)*
+ fn parseBoolOrExpr(p: *Parser) Error!Node.Index {
+ var res = try p.parseBoolAndExpr();
+ if (res == 0) return null_node;
- /// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)*
- fn parseBitShiftExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseBitShiftOp, parseAdditionExpr, .Infinitely);
+ while (true) {
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_or => {
+ const or_token = p.nextToken();
+ const rhs = try p.parseBoolAndExpr();
+ if (rhs == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ }
+ res = try p.addNode(.{
+ .tag = .BoolOr,
+ .main_token = or_token,
+ .data = .{
+ .lhs = res,
+ .rhs = rhs,
+ },
+ });
+ },
+ else => return res,
+ }
+ }
}
- /// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)*
- fn parseAdditionExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseAdditionOp, parseMultiplyExpr, .Infinitely);
- }
+ /// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)*
+ fn parseBoolAndExpr(p: *Parser) !Node.Index {
+ var res = try p.parseCompareExpr();
+ if (res == 0) return null_node;
- /// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)*
- fn parseMultiplyExpr(p: *Parser) !?*Node {
- return p.parseBinOpExpr(parseMultiplyOp, parsePrefixExpr, .Infinitely);
+ while (true) {
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_and => {
+ const and_token = p.nextToken();
+ const rhs = try p.parseCompareExpr();
+ if (rhs == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ }
+ res = try p.addNode(.{
+ .tag = .BoolAnd,
+ .main_token = and_token,
+ .data = .{
+ .lhs = res,
+ .rhs = rhs,
+ },
+ });
+ },
+ else => return res,
+ }
+ }
}
- /// PrefixExpr <- PrefixOp* PrimaryExpr
- fn parsePrefixExpr(p: *Parser) !?*Node {
- return p.parsePrefixOpExpr(parsePrefixOp, parsePrimaryExpr);
+ /// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)?
+ /// CompareOp
+ /// <- EQUALEQUAL
+ /// / EXCLAMATIONMARKEQUAL
+ /// / LARROW
+ /// / RARROW
+ /// / LARROWEQUAL
+ /// / RARROWEQUAL
+ fn parseCompareExpr(p: *Parser) !Node.Index {
+ const expr = try p.parseBitwiseExpr();
+ if (expr == 0) return null_node;
+
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .EqualEqual => .EqualEqual,
+ .BangEqual => .BangEqual,
+ .AngleBracketLeft => .LessThan,
+ .AngleBracketRight => .GreaterThan,
+ .AngleBracketLeftEqual => .LessOrEqual,
+ .AngleBracketRightEqual => .GreaterOrEqual,
+ else => return expr,
+ };
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = expr,
+ .rhs = try p.expectBitwiseExpr(),
+ },
+ });
}
- /// PrimaryExpr
- /// <- AsmExpr
- /// / IfExpr
- /// / KEYWORD_break BreakLabel? Expr?
- /// / KEYWORD_comptime Expr
- /// / KEYWORD_nosuspend Expr
- /// / KEYWORD_continue BreakLabel?
- /// / KEYWORD_resume Expr
- /// / KEYWORD_return Expr?
- /// / BlockLabel? LoopExpr
- /// / Block
- /// / CurlySuffixExpr
- fn parsePrimaryExpr(p: *Parser) !?*Node {
- if (try p.parseAsmExpr()) |node| return node;
- if (try p.parseIfExpr()) |node| return node;
-
- if (p.eatToken(.Keyword_break)) |token| {
- const label = try p.parseBreakLabel();
- const expr_node = try p.parseExpr();
- const node = try Node.ControlFlowExpression.create(&p.arena.allocator, .{
- .tag = .Break,
- .ltoken = token,
- }, .{
- .label = label,
- .rhs = expr_node,
- });
- return &node.base;
- }
+ /// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)*
+ /// BitwiseOp
+ /// <- AMPERSAND
+ /// / CARET
+ /// / PIPE
+ /// / KEYWORD_orelse
+ /// / KEYWORD_catch Payload?
+ fn parseBitwiseExpr(p: *Parser) !Node.Index {
+ var res = try p.parseBitShiftExpr();
+ if (res == 0) return null_node;
- if (p.eatToken(.Keyword_comptime)) |token| {
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- const node = try p.arena.allocator.create(Node.Comptime);
- node.* = .{
- .doc_comments = null,
- .comptime_token = token,
- .expr = expr_node,
+ while (true) {
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .Ampersand => .BitAnd,
+ .Caret => .BitXor,
+ .Pipe => .BitOr,
+ .Keyword_orelse => .OrElse,
+ .Keyword_catch => {
+ const catch_token = p.nextToken();
+ _ = try p.parsePayload();
+ const rhs = try p.parseBitShiftExpr();
+ if (rhs == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ }
+ res = try p.addNode(.{
+ .tag = .Catch,
+ .main_token = catch_token,
+ .data = .{
+ .lhs = res,
+ .rhs = rhs,
+ },
+ });
+ continue;
+ },
+ else => return res,
};
- return &node.base;
- }
-
- if (p.eatToken(.Keyword_nosuspend)) |token| {
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
+ res = try p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = res,
+ .rhs = try p.expectBitShiftExpr(),
+ },
});
- const node = try p.arena.allocator.create(Node.Nosuspend);
- node.* = .{
- .nosuspend_token = token,
- .expr = expr_node,
- };
- return &node.base;
}
+ }
- if (p.eatToken(.Keyword_continue)) |token| {
- const label = try p.parseBreakLabel();
- const node = try Node.ControlFlowExpression.create(&p.arena.allocator, .{
- .tag = .Continue,
- .ltoken = token,
- }, .{
- .label = label,
- .rhs = null,
- });
- return &node.base;
+ fn expectBitwiseExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseBitwiseExpr();
+ if (node == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ } else {
+ return node;
}
+ }
- if (p.eatToken(.Keyword_resume)) |token| {
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- const node = try p.arena.allocator.create(Node.SimplePrefixOp);
- node.* = .{
- .base = .{ .tag = .Resume },
- .op_token = token,
- .rhs = expr_node,
- };
- return &node.base;
- }
+ /// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)*
+ /// BitShiftOp
+ /// <- LARROW2
+ /// / RARROW2
+ fn parseBitShiftExpr(p: *Parser) Error!Node.Index {
+ var res = try p.parseAdditionExpr();
+ if (res == 0) return null_node;
- if (p.eatToken(.Keyword_return)) |token| {
- const expr_node = try p.parseExpr();
- const node = try Node.ControlFlowExpression.create(&p.arena.allocator, .{
- .tag = .Return,
- .ltoken = token,
- }, .{
- .rhs = expr_node,
+ while (true) {
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .AngleBracketAngleBracketLeft => .BitShiftLeft,
+ .AngleBracketAngleBracketRight => .BitShiftRight,
+ else => return res,
+ };
+ res = try p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = res,
+ .rhs = try p.expectAdditionExpr(),
+ },
});
- return &node.base;
}
+ }
- var colon: TokenIndex = undefined;
- const label = p.parseBlockLabel(&colon);
- if (try p.parseLoopExpr()) |node| {
- if (node.cast(Node.For)) |for_node| {
- for_node.label = label;
- } else if (node.cast(Node.While)) |while_node| {
- while_node.label = label;
- } else unreachable;
+ fn expectBitShiftExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseBitShiftExpr();
+ if (node == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+ } else {
return node;
}
- if (label) |token| {
- p.putBackToken(token + 1); // ":"
- p.putBackToken(token); // IDENTIFIER
- }
-
- if (try p.parseBlock(null)) |node| return node;
- if (try p.parseCurlySuffixExpr()) |node| return node;
-
- return null;
- }
-
- /// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)?
- fn parseIfExpr(p: *Parser) !?*Node {
- return p.parseIf(parseExpr);
}
- /// Block <- LBRACE Statement* RBRACE
- fn parseBlock(p: *Parser, label_token: ?TokenIndex) !?*Node {
- const lbrace = p.eatToken(.LBrace) orelse return null;
-
- var statements = std.ArrayList(*Node).init(p.gpa);
- defer statements.deinit();
+ /// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)*
+ /// AdditionOp
+ /// <- PLUS
+ /// / MINUS
+ /// / PLUS2
+ /// / PLUSPERCENT
+ /// / MINUSPERCENT
+ fn parseAdditionExpr(p: *Parser) Error!Node.Index {
+ var res = try p.parseMultiplyExpr();
+ if (res == 0) return null_node;
while (true) {
- const statement = (p.parseStatement() catch |err| switch (err) {
- error.OutOfMemory => return error.OutOfMemory,
- error.ParseError => {
- // try to skip to the next statement
- p.findNextStmt();
- continue;
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .Plus => .Add,
+ .Minus => .Sub,
+ .PlusPlus => .ArrayCat,
+ .PlusPercent => .AddWrap,
+ .MinusPercent => .SubWrap,
+ else => return res,
+ };
+ res = try p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = res,
+ .rhs = try p.expectMultiplyExpr(),
},
- }) orelse break;
- try statements.append(statement);
+ });
}
+ }
- const rbrace = try p.expectToken(.RBrace);
-
- const statements_len = @intCast(NodeIndex, statements.items.len);
-
- if (label_token) |label| {
- const block_node = try Node.LabeledBlock.alloc(&p.arena.allocator, statements_len);
- block_node.* = .{
- .label = label,
- .lbrace = lbrace,
- .statements_len = statements_len,
- .rbrace = rbrace,
- };
- std.mem.copy(*Node, block_node.statements(), statements.items);
- return &block_node.base;
- } else {
- const block_node = try Node.Block.alloc(&p.arena.allocator, statements_len);
- block_node.* = .{
- .lbrace = lbrace,
- .statements_len = statements_len,
- .rbrace = rbrace,
- };
- std.mem.copy(*Node, block_node.statements(), statements.items);
- return &block_node.base;
+ fn expectAdditionExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseAdditionExpr();
+ if (node == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
}
+ return node;
}
- /// LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr)
- fn parseLoopExpr(p: *Parser) !?*Node {
- const inline_token = p.eatToken(.Keyword_inline);
+ /// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)*
+ /// MultiplyOp
+ /// <- PIPE2
+ /// / ASTERISK
+ /// / SLASH
+ /// / PERCENT
+ /// / ASTERISK2
+ /// / ASTERISKPERCENT
+ fn parseMultiplyExpr(p: *Parser) Error!Node.Index {
+ var res = try p.parsePrefixExpr();
+ if (res == 0) return null_node;
- if (try p.parseForExpr()) |node| {
- node.cast(Node.For).?.inline_token = inline_token;
- return node;
+ while (true) {
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .PipePipe => .MergeErrorSets,
+ .Asterisk => .Mul,
+ .Slash => .Div,
+ .Percent => .Mod,
+ .AsteriskAsterisk => .ArrayMult,
+ .AsteriskPercent => .MulWrap,
+ else => return res,
+ };
+ res = try p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = res,
+ .rhs = try p.expectPrefixExpr(),
+ },
+ });
}
+ }
- if (try p.parseWhileExpr()) |node| {
- node.cast(Node.While).?.inline_token = inline_token;
- return node;
+ fn expectMultiplyExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseMultiplyExpr();
+ if (node == 0) {
+ return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
}
-
- if (inline_token == null) return null;
-
- // If we've seen "inline", there should have been a "for" or "while"
- try p.errors.append(p.gpa, .{
- .ExpectedInlinable = .{ .token = p.tok_i },
- });
- return error.ParseError;
+ return node;
}
- /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)?
- fn parseForExpr(p: *Parser) !?*Node {
- const node = (try p.parseForPrefix()) orelse return null;
- const for_prefix = node.cast(Node.For).?;
-
- const body_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
+ /// PrefixExpr <- PrefixOp* PrimaryExpr
+ /// PrefixOp
+ /// <- EXCLAMATIONMARK
+ /// / MINUS
+ /// / TILDE
+ /// / MINUSPERCENT
+ /// / AMPERSAND
+ /// / KEYWORD_try
+ /// / KEYWORD_await
+ fn parsePrefixExpr(p: *Parser) Error!Node.Index {
+ const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
+ .Bang => .BoolNot,
+ .Minus => .Negation,
+ .Tilde => .BitNot,
+ .MinusPercent => .NegationWrap,
+ .Ampersand => .AddressOf,
+ .Keyword_try => .Try,
+ .Keyword_await => .Await,
+ else => return p.parsePrimaryExpr(),
+ };
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.expectPrefixExpr(),
+ .rhs = undefined,
+ },
});
- for_prefix.body = body_node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const body = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = null,
- .body = body,
- };
+ }
- for_prefix.@"else" = else_node;
+ fn expectPrefixExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parsePrefixExpr();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedPrefixExpr = .{ .token = p.tok_i } });
}
-
return node;
}
- /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)?
- fn parseWhileExpr(p: *Parser) !?*Node {
- const node = (try p.parseWhilePrefix()) orelse return null;
- const while_prefix = node.cast(Node.While).?;
-
- const body_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- while_prefix.body = body_node;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const payload = try p.parsePayload();
- const body = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = payload,
- .body = body,
- };
-
- while_prefix.@"else" = else_node;
+ /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr
+ /// PrefixTypeOp
+ /// <- QUESTIONMARK
+ /// / KEYWORD_anyframe MINUSRARROW
+ /// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
+ /// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
+ /// PtrTypeStart
+ /// <- ASTERISK
+ /// / ASTERISK2
+ /// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
+ /// ArrayTypeStart <- LBRACKET Expr? (COLON Expr)? RBRACKET
+ fn parseTypeExpr(p: *Parser) Error!Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .QuestionMark => return p.addNode(.{
+ .tag = .OptionalType,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.expectTypeExpr(),
+ .rhs = undefined,
+ },
+ }),
+ .Keyword_anyframe => switch (p.token_tags[p.tok_i + 1]) {
+ .Arrow => return p.addNode(.{
+ .tag = .AnyFrameType,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = p.nextToken(),
+ .rhs = try p.expectTypeExpr(),
+ },
+ }),
+ else => return p.parseErrorUnionExpr(),
+ },
+ .Asterisk => {
+ const asterisk = p.nextToken();
+ const mods = try p.parsePtrModifiers();
+ const elem_type = try p.expectTypeExpr();
+ if (mods.bit_range_start == 0) {
+ return p.addNode(.{
+ .tag = .PtrTypeAligned,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = mods.align_node,
+ .rhs = elem_type,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .PtrType,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = try p.addExtra(Node.PtrType{
+ .sentinel = 0,
+ .align_node = mods.align_node,
+ .bit_range_start = mods.bit_range_start,
+ .bit_range_end = mods.bit_range_end,
+ }),
+ .rhs = elem_type,
+ },
+ });
+ }
+ },
+ .AsteriskAsterisk => {
+ const asterisk = p.nextToken();
+ const mods = try p.parsePtrModifiers();
+ const elem_type = try p.expectTypeExpr();
+ const inner: Node.Index = inner: {
+ if (mods.bit_range_start == 0) {
+ break :inner try p.addNode(.{
+ .tag = .PtrTypeAligned,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = mods.align_node,
+ .rhs = elem_type,
+ },
+ });
+ } else {
+ break :inner try p.addNode(.{
+ .tag = .PtrType,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = try p.addExtra(Node.PtrType{
+ .sentinel = 0,
+ .align_node = mods.align_node,
+ .bit_range_start = mods.bit_range_start,
+ .bit_range_end = mods.bit_range_end,
+ }),
+ .rhs = elem_type,
+ },
+ });
+ }
+ };
+ return p.addNode(.{
+ .tag = .PtrTypeAligned,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = 0,
+ .rhs = inner,
+ },
+ });
+ },
+ .LBracket => switch (p.token_tags[p.tok_i + 1]) {
+ .Asterisk => {
+ const lbracket = p.nextToken();
+ const asterisk = p.nextToken();
+ var sentinel: Node.Index = 0;
+ prefix: {
+ if (p.eatToken(.Identifier)) |ident| {
+ const token_slice = p.source[p.token_starts[ident]..][0..2];
+ if (!std.mem.eql(u8, token_slice, "c]")) {
+ p.tok_i -= 1;
+ } else {
+ break :prefix;
+ }
+ }
+ if (p.eatToken(.Colon)) |_| {
+ sentinel = try p.expectExpr();
+ }
+ }
+ _ = try p.expectToken(.RBracket);
+ const mods = try p.parsePtrModifiers();
+ const elem_type = try p.expectTypeExpr();
+ if (mods.bit_range_start == 0) {
+ if (sentinel == 0) {
+ return p.addNode(.{
+ .tag = .PtrTypeAligned,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = mods.align_node,
+ .rhs = elem_type,
+ },
+ });
+ } else if (mods.align_node == 0) {
+ return p.addNode(.{
+ .tag = .PtrTypeSentinel,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = sentinel,
+ .rhs = elem_type,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .SliceType,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = try p.addExtra(.{
+ .sentinel = sentinel,
+ .align_node = mods.align_node,
+ }),
+ .rhs = elem_type,
+ },
+ });
+ }
+ } else {
+ return p.addNode(.{
+ .tag = .PtrType,
+ .main_token = asterisk,
+ .data = .{
+ .lhs = try p.addExtra(.{
+ .sentinel = sentinel,
+ .align_node = mods.align_node,
+ .bit_range_start = mods.bit_range_start,
+ .bit_range_end = mods.bit_range_end,
+ }),
+ .rhs = elem_type,
+ },
+ });
+ }
+ },
+ else => {
+ const lbracket = p.nextToken();
+ const len_expr = try p.parseExpr();
+ const sentinel: Node.Index = if (p.eatToken(.Colon)) |_|
+ try p.expectExpr()
+ else
+ 0;
+ _ = try p.expectToken(.RBracket);
+ const mods = try p.parsePtrModifiers();
+ const elem_type = try p.expectTypeExpr();
+ if (mods.bit_range_start != 0) {
+ @panic("TODO implement this error");
+ //try p.warn(.{
+ // .BitRangeInvalid = .{ .node = mods.bit_range_start },
+ //});
+ }
+ if (len_expr == 0) {
+ if (sentinel == 0) {
+ return p.addNode(.{
+ .tag = .PtrTypeAligned,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = mods.align_node,
+ .rhs = elem_type,
+ },
+ });
+ } else if (mods.align_node == 0) {
+ return p.addNode(.{
+ .tag = .PtrTypeSentinel,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = sentinel,
+ .rhs = elem_type,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .SliceType,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = try p.addExtra(.{
+ .sentinel = sentinel,
+ .align_node = mods.align_node,
+ }),
+ .rhs = elem_type,
+ },
+ });
+ }
+ } else {
+ if (mods.align_node != 0) {
+ @panic("TODO implement this error");
+ //try p.warn(.{
+ // .AlignInvalid = .{ .node = mods.align_node },
+ //});
+ }
+ if (sentinel == 0) {
+ return p.addNode(.{
+ .tag = .ArrayType,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = len_expr,
+ .rhs = elem_type,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .ArrayTypeSentinel,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = len_expr,
+ .rhs = try p.addExtra(.{
+ .elem_type = elem_type,
+ .sentinel = sentinel,
+ }),
+ },
+ });
+ }
+ }
+ },
+ },
+ else => return p.parseErrorUnionExpr(),
}
+ }
+ fn expectTypeExpr(p: *Parser) Error!Node.Index {
+ const node = try p.parseTypeExpr();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedTypeExpr = .{ .token = p.tok_i } });
+ }
return node;
}
- /// CurlySuffixExpr <- TypeExpr InitList?
- fn parseCurlySuffixExpr(p: *Parser) !?*Node {
- const lhs = (try p.parseTypeExpr()) orelse return null;
- const suffix_op = (try p.parseInitList(lhs)) orelse return lhs;
- return suffix_op;
+ /// PrimaryExpr
+ /// <- AsmExpr
+ /// / IfExpr
+ /// / KEYWORD_break BreakLabel? Expr?
+ /// / KEYWORD_comptime Expr
+ /// / KEYWORD_nosuspend Expr
+ /// / KEYWORD_continue BreakLabel?
+ /// / KEYWORD_resume Expr
+ /// / KEYWORD_return Expr?
+ /// / BlockLabel? LoopExpr
+ /// / Block
+ /// / CurlySuffixExpr
+ fn parsePrimaryExpr(p: *Parser) !Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_asm => return p.parseAsmExpr(),
+ .Keyword_if => return p.parseIfExpr(),
+ .Keyword_break => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Break,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.parseBreakLabel(),
+ .rhs = try p.parseExpr(),
+ },
+ });
+ },
+ .Keyword_continue => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Continue,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.parseBreakLabel(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_comptime => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Comptime,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.expectExpr(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_nosuspend => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Nosuspend,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.expectExpr(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_resume => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Resume,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.expectExpr(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Keyword_return => {
+ p.tok_i += 1;
+ return p.addNode(.{
+ .tag = .Return,
+ .main_token = p.tok_i - 1,
+ .data = .{
+ .lhs = try p.parseExpr(),
+ .rhs = undefined,
+ },
+ });
+ },
+ .Identifier => {
+ if (p.token_tags[p.tok_i + 1] == .Colon) {
+ switch (p.token_tags[p.tok_i + 2]) {
+ .Keyword_inline => {
+ p.tok_i += 3;
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_for => return p.parseForExpr(),
+ .Keyword_while => return p.parseWhileExpr(),
+ else => return p.fail(.{
+ .ExpectedInlinable = .{ .token = p.tok_i },
+ }),
+ }
+ },
+ .Keyword_for => {
+ p.tok_i += 2;
+ return p.parseForExpr();
+ },
+ .Keyword_while => {
+ p.tok_i += 2;
+ return p.parseWhileExpr();
+ },
+ .LBrace => {
+ p.tok_i += 2;
+ return p.parseBlock();
+ },
+ else => return p.parseCurlySuffixExpr(),
+ }
+ } else {
+ return p.parseCurlySuffixExpr();
+ }
+ },
+ .Keyword_inline => {
+ p.tok_i += 2;
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_for => return p.parseForExpr(),
+ .Keyword_while => return p.parseWhileExpr(),
+ else => return p.fail(.{
+ .ExpectedInlinable = .{ .token = p.tok_i },
+ }),
+ }
+ },
+ .Keyword_for => return p.parseForExpr(),
+ .Keyword_while => return p.parseWhileExpr(),
+ .LBrace => return p.parseBlock(),
+ else => return p.parseCurlySuffixExpr(),
+ }
}
- /// InitList
- /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
- /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
- /// / LBRACE RBRACE
- fn parseInitList(p: *Parser, lhs: *Node) !?*Node {
- const lbrace = p.eatToken(.LBrace) orelse return null;
- var init_list = std.ArrayList(*Node).init(p.gpa);
- defer init_list.deinit();
+ /// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)?
+ fn parseIfExpr(p: *Parser) !Node.Index {
+ return p.parseIf(parseExpr);
+ }
- if (try p.parseFieldInit()) |field_init| {
- try init_list.append(field_init);
- while (p.eatToken(.Comma)) |_| {
- const next = (try p.parseFieldInit()) orelse break;
- try init_list.append(next);
- }
- const node = try Node.StructInitializer.alloc(&p.arena.allocator, init_list.items.len);
- node.* = .{
- .lhs = lhs,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = init_list.items.len,
- };
- std.mem.copy(*Node, node.list(), init_list.items);
- return &node.base;
- }
+ /// Block <- LBRACE Statement* RBRACE
+ fn parseBlock(p: *Parser) !Node.Index {
+ const lbrace = p.eatToken(.LBrace) orelse return null_node;
- if (try p.parseExpr()) |expr| {
- try init_list.append(expr);
- while (p.eatToken(.Comma)) |_| {
- const next = (try p.parseExpr()) orelse break;
- try init_list.append(next);
- }
- const node = try Node.ArrayInitializer.alloc(&p.arena.allocator, init_list.items.len);
- node.* = .{
- .lhs = lhs,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = init_list.items.len,
- };
- std.mem.copy(*Node, node.list(), init_list.items);
- return &node.base;
+ var statements = std.ArrayList(Node.Index).init(p.gpa);
+ defer statements.deinit();
+
+ while (true) {
+ const statement = (p.parseStatement() catch |err| switch (err) {
+ error.OutOfMemory => return error.OutOfMemory,
+ error.ParseError => {
+ // try to skip to the next statement
+ p.findNextStmt();
+ continue;
+ },
+ });
+ if (statement == 0) break;
+ try statements.append(statement);
}
- const node = try p.arena.allocator.create(Node.StructInitializer);
- node.* = .{
- .lhs = lhs,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = 0,
+ const rbrace = try p.expectToken(.RBrace);
+ const statements_span = try p.listToSpan(statements.items);
+
+ return p.addNode(.{
+ .tag = .Block,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = statements_span.start,
+ .rhs = statements_span.end,
+ },
+ });
+ }
+
+ /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
+ /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)?
+ fn parseForExpr(p: *Parser) !Node.Index {
+ const for_token = p.eatToken(.Keyword_for) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const array_expr = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ _ = try p.parsePtrIndexPayload();
+
+ const then_expr = try p.expectExpr();
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ return p.addNode(.{
+ .tag = .ForSimple,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = then_expr,
+ },
+ });
+ };
+ const else_expr = try p.expectExpr();
+ return p.addNode(.{
+ .tag = .For,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = try p.addExtra(Node.If{
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
+ });
+ }
+
+ /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
+ /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)?
+ fn parseWhileExpr(p: *Parser) !Node.Index {
+ const while_token = p.eatToken(.Keyword_while) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const condition = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ const then_payload = try p.parsePtrPayload();
+ const continue_expr = try p.parseWhileContinueExpr();
+
+ const then_expr = try p.expectExpr();
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ if (continue_expr == 0) {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileSimple else .WhileSimpleOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = then_expr,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileCont else .WhileContOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.WhileCont{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ }),
+ },
+ });
+ }
};
- return &node.base;
+ const else_payload = try p.parsePayload();
+ const else_expr = try p.expectExpr();
+ const tag = if (else_payload != 0)
+ Node.Tag.WhileError
+ else if (then_payload != 0)
+ Node.Tag.WhileOptional
+ else
+ Node.Tag.While;
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.While{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
+ });
}
+ /// CurlySuffixExpr <- TypeExpr InitList?
/// InitList
/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
/// / LBRACE RBRACE
- fn parseAnonInitList(p: *Parser, dot: TokenIndex) !?*Node {
- const lbrace = p.eatToken(.LBrace) orelse return null;
- var init_list = std.ArrayList(*Node).init(p.gpa);
- defer init_list.deinit();
+ fn parseCurlySuffixExpr(p: *Parser) !Node.Index {
+ const lhs = try p.parseTypeExpr();
+ if (lhs == 0) return null_node;
+ const lbrace = p.eatToken(.LBrace) orelse return lhs;
+
+ // If there are 0 or 1 items, we can use ArrayInitOne/StructInitOne;
+ // otherwise we use the full ArrayInit/StructInit.
+
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .StructInitOne,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = lhs,
+ .rhs = 0,
+ },
+ });
+ }
+ const field_init = try p.parseFieldInit();
+ if (field_init != 0) {
+ const comma_one = p.eatToken(.Comma);
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .StructInitOne,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = lhs,
+ .rhs = field_init,
+ },
+ });
+ }
+
+ var init_list = std.ArrayList(Node.Index).init(p.gpa);
+ defer init_list.deinit();
- if (try p.parseFieldInit()) |field_init| {
try init_list.append(field_init);
- while (p.eatToken(.Comma)) |_| {
- const next = (try p.parseFieldInit()) orelse break;
- try init_list.append(next);
- }
- const node = try Node.StructInitializerDot.alloc(&p.arena.allocator, init_list.items.len);
- node.* = .{
- .dot = dot,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = init_list.items.len,
- };
- std.mem.copy(*Node, node.list(), init_list.items);
- return &node.base;
- }
- if (try p.parseExpr()) |expr| {
- try init_list.append(expr);
- while (p.eatToken(.Comma)) |_| {
- const next = (try p.parseExpr()) orelse break;
+ while (true) {
+ const next = try p.expectFieldInit();
try init_list.append(next);
+
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => {
+ if (p.eatToken(.RBrace)) |_| break;
+ continue;
+ },
+ .RBrace => break,
+ .Colon, .RParen, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .RBrace,
+ },
+ });
+ },
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
}
- const node = try Node.ArrayInitializerDot.alloc(&p.arena.allocator, init_list.items.len);
- node.* = .{
- .dot = dot,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = init_list.items.len,
- };
- std.mem.copy(*Node, node.list(), init_list.items);
- return &node.base;
+ const span = try p.listToSpan(init_list.items);
+ return p.addNode(.{
+ .tag = .StructInit,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = lhs,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = span.start,
+ .end = span.end,
+ }),
+ },
+ });
}
- const node = try p.arena.allocator.create(Node.StructInitializerDot);
- node.* = .{
- .dot = dot,
- .rtoken = try p.expectToken(.RBrace),
- .list_len = 0,
- };
- return &node.base;
- }
+ const elem_init = try p.expectExpr();
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .ArrayInitOne,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = lhs,
+ .rhs = elem_init,
+ },
+ });
+ }
- /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr
- fn parseTypeExpr(p: *Parser) Error!?*Node {
- return p.parsePrefixOpExpr(parsePrefixTypeOp, parseErrorUnionExpr);
- }
+ var init_list = std.ArrayList(Node.Index).init(p.gpa);
+ defer init_list.deinit();
- /// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)?
- fn parseErrorUnionExpr(p: *Parser) !?*Node {
- const suffix_expr = (try p.parseSuffixExpr()) orelse return null;
+ try init_list.append(elem_init);
- if (try SimpleBinOpParseFn(.Bang, .ErrorUnion)(p)) |node| {
- const error_union = node.castTag(.ErrorUnion).?;
- const type_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
- });
- error_union.lhs = suffix_expr;
- error_union.rhs = type_expr;
- return node;
+ while (p.eatToken(.Comma)) |_| {
+ const next = try p.parseExpr();
+ if (next == 0) break;
+ try init_list.append(next);
}
+ _ = try p.expectToken(.RBrace);
+ const span = try p.listToSpan(init_list.items);
+ return p.addNode(.{
+ .tag = .ArrayInit,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = lhs,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = span.start,
+ .end = span.end,
+ }),
+ },
+ });
+ }
- return suffix_expr;
+ /// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)?
+ fn parseErrorUnionExpr(p: *Parser) !Node.Index {
+ const suffix_expr = try p.parseSuffixExpr();
+ if (suffix_expr == 0) return null_node;
+ const bang = p.eatToken(.Bang) orelse return suffix_expr;
+ return p.addNode(.{
+ .tag = .ErrorUnion,
+ .main_token = bang,
+ .data = .{
+ .lhs = suffix_expr,
+ .rhs = try p.expectTypeExpr(),
+ },
+ });
}
/// SuffixExpr
/// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
/// / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
- fn parseSuffixExpr(p: *Parser) !?*Node {
- const maybe_async = p.eatToken(.Keyword_async);
- if (maybe_async) |async_token| {
- const token_fn = p.eatToken(.Keyword_fn);
- if (token_fn != null) {
- // TODO: remove this hack when async fn rewriting is
- // HACK: If we see the keyword `fn`, then we assume that
- // we are parsing an async fn proto, and not a call.
- // We therefore put back all tokens consumed by the async
- // prefix...
- p.putBackToken(token_fn.?);
- p.putBackToken(async_token);
- return p.parsePrimaryTypeExpr();
- }
- var res = try p.expectNode(parsePrimaryTypeExpr, .{
- .ExpectedPrimaryTypeExpr = .{ .token = p.tok_i },
- });
+ /// FnCallArguments <- LPAREN ExprList RPAREN
+ /// ExprList <- (Expr COMMA)* Expr?
+ /// TODO detect when there is 1 or less parameter to the call and emit
+ /// CallOne instead of Call.
+ fn parseSuffixExpr(p: *Parser) !Node.Index {
+ if (p.eatToken(.Keyword_async)) |async_token| {
+ var res = try p.expectPrimaryTypeExpr();
- while (try p.parseSuffixOp(res)) |node| {
+ while (true) {
+ const node = try p.parseSuffixOp(res);
+ if (node == 0) break;
res = node;
}
-
- const params = (try p.parseFnCallArguments()) orelse {
- try p.errors.append(p.gpa, .{
- .ExpectedParamList = .{ .token = p.tok_i },
- });
- // ignore this, continue parsing
+ const lparen = (try p.expectTokenRecoverable(.LParen)) orelse {
+ try p.warn(.{ .ExpectedParamList = .{ .token = p.tok_i } });
return res;
};
- defer p.gpa.free(params.list);
- const node = try Node.Call.alloc(&p.arena.allocator, params.list.len);
- node.* = .{
- .lhs = res,
- .params_len = params.list.len,
- .async_token = async_token,
- .rtoken = params.rparen,
- };
- std.mem.copy(*Node, node.params(), params.list);
- return &node.base;
+ const params = try ListParseFn(parseExpr)(p);
+ _ = try p.expectToken(.RParen);
+
+ return p.addNode(.{
+ .tag = .Call,
+ .main_token = lparen,
+ .data = .{
+ .lhs = res,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = params.start,
+ .end = params.end,
+ }),
+ },
+ });
}
- if (try p.parsePrimaryTypeExpr()) |expr| {
- var res = expr;
+ var res = try p.parsePrimaryTypeExpr();
+ if (res == 0) return res;
- while (true) {
- if (try p.parseSuffixOp(res)) |node| {
- res = node;
- continue;
- }
- if (try p.parseFnCallArguments()) |params| {
- defer p.gpa.free(params.list);
- const call = try Node.Call.alloc(&p.arena.allocator, params.list.len);
- call.* = .{
- .lhs = res,
- .params_len = params.list.len,
- .async_token = null,
- .rtoken = params.rparen,
- };
- std.mem.copy(*Node, call.params(), params.list);
- res = &call.base;
- continue;
- }
- break;
+ while (true) {
+ const suffix_op = try p.parseSuffixOp(res);
+ if (suffix_op != 0) {
+ res = suffix_op;
+ continue;
}
- return res;
+ const lparen = p.eatToken(.LParen) orelse return res;
+ const params = try ListParseFn(parseExpr)(p);
+ _ = try p.expectToken(.RParen);
+
+ res = try p.addNode(.{
+ .tag = .Call,
+ .main_token = lparen,
+ .data = .{
+ .lhs = res,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = params.start,
+ .end = params.end,
+ }),
+ },
+ });
}
-
- return null;
}
/// PrimaryTypeExpr
@@ -1521,6 +2196,7 @@ const Parser = struct {
/// / CHAR_LITERAL
/// / ContainerDecl
/// / DOT IDENTIFIER
+ /// / DOT InitList
/// / ErrorSetDecl
/// / FLOAT
/// / FnProto
@@ -1539,260 +2215,497 @@ const Parser = struct {
/// / KEYWORD_unreachable
/// / STRINGLITERAL
/// / SwitchExpr
- fn parsePrimaryTypeExpr(p: *Parser) !?*Node {
- if (try p.parseBuiltinCall()) |node| return node;
- if (p.eatToken(.CharLiteral)) |token| {
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .CharLiteral },
- .token = token,
- };
- return &node.base;
- }
- if (try p.parseContainerDecl()) |node| return node;
- if (try p.parseAnonLiteral()) |node| return node;
- if (try p.parseErrorSetDecl()) |node| return node;
- if (try p.parseFloatLiteral()) |node| return node;
- if (try p.parseFnProto(.as_type, .{})) |node| return node;
- if (try p.parseGroupedExpr()) |node| return node;
- if (try p.parseLabeledTypeExpr()) |node| return node;
- if (try p.parseIdentifier()) |node| return node;
- if (try p.parseIfTypeExpr()) |node| return node;
- if (try p.parseIntegerLiteral()) |node| return node;
- if (p.eatToken(.Keyword_comptime)) |token| {
- const expr = (try p.parseTypeExpr()) orelse return null;
- const node = try p.arena.allocator.create(Node.Comptime);
- node.* = .{
- .doc_comments = null,
- .comptime_token = token,
- .expr = expr,
- };
- return &node.base;
- }
- if (p.eatToken(.Keyword_error)) |token| {
- const period = try p.expectTokenRecoverable(.Period);
- const identifier = try p.expectNodeRecoverable(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
- const global_error_set = try p.createLiteral(.ErrorType, token);
- if (period == null or identifier == null) return global_error_set;
-
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = Node{ .tag = .Period },
- .op_token = period.?,
- .lhs = global_error_set,
- .rhs = identifier.?,
- };
- return &node.base;
- }
- if (p.eatToken(.Keyword_false)) |token| return p.createLiteral(.BoolLiteral, token);
- if (p.eatToken(.Keyword_null)) |token| return p.createLiteral(.NullLiteral, token);
- if (p.eatToken(.Keyword_anyframe)) |token| {
- const node = try p.arena.allocator.create(Node.AnyFrameType);
- node.* = .{
- .anyframe_token = token,
- .result = null,
- };
- return &node.base;
- }
- if (p.eatToken(.Keyword_true)) |token| return p.createLiteral(.BoolLiteral, token);
- if (p.eatToken(.Keyword_undefined)) |token| return p.createLiteral(.UndefinedLiteral, token);
- if (p.eatToken(.Keyword_unreachable)) |token| return p.createLiteral(.Unreachable, token);
- if (try p.parseStringLiteral()) |node| return node;
- if (try p.parseSwitchExpr()) |node| return node;
-
- return null;
- }
-
/// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
- fn parseContainerDecl(p: *Parser) !?*Node {
- const layout_token = p.eatToken(.Keyword_extern) orelse
- p.eatToken(.Keyword_packed);
-
- const node = (try p.parseContainerDeclAuto()) orelse {
- if (layout_token) |token|
- p.putBackToken(token);
- return null;
- };
- node.cast(Node.ContainerDecl).?.*.layout_token = layout_token;
- return node;
- }
-
- /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
- fn parseErrorSetDecl(p: *Parser) !?*Node {
- const error_token = p.eatToken(.Keyword_error) orelse return null;
- if (p.eatToken(.LBrace) == null) {
- // Might parse as `KEYWORD_error DOT IDENTIFIER` later in PrimaryTypeExpr, so don't error
- p.putBackToken(error_token);
- return null;
- }
- const decls = try p.parseErrorTagList();
- defer p.gpa.free(decls);
- const rbrace = try p.expectToken(.RBrace);
-
- const node = try Node.ErrorSetDecl.alloc(&p.arena.allocator, decls.len);
- node.* = .{
- .error_token = error_token,
- .decls_len = decls.len,
- .rbrace_token = rbrace,
- };
- std.mem.copy(*Node, node.decls(), decls);
- return &node.base;
- }
-
+ /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
+ /// InitList
+ /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE
+ /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE
+ /// / LBRACE RBRACE
+ /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE
/// GroupedExpr <- LPAREN Expr RPAREN
- fn parseGroupedExpr(p: *Parser) !?*Node {
- const lparen = p.eatToken(.LParen) orelse return null;
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- const rparen = try p.expectToken(.RParen);
-
- const node = try p.arena.allocator.create(Node.GroupedExpression);
- node.* = .{
- .lparen = lparen,
- .expr = expr,
- .rparen = rparen,
- };
- return &node.base;
- }
-
/// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
- fn parseIfTypeExpr(p: *Parser) !?*Node {
- return p.parseIf(parseTypeExpr);
- }
-
/// LabeledTypeExpr
/// <- BlockLabel Block
/// / BlockLabel? LoopTypeExpr
- fn parseLabeledTypeExpr(p: *Parser) !?*Node {
- var colon: TokenIndex = undefined;
- const label = p.parseBlockLabel(&colon);
+ /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
+ fn parsePrimaryTypeExpr(p: *Parser) !Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .CharLiteral,
+ .IntegerLiteral,
+ .FloatLiteral,
+ .StringLiteral,
+ .Keyword_false,
+ .Keyword_true,
+ .Keyword_null,
+ .Keyword_undefined,
+ .Keyword_unreachable,
+ .Keyword_anyframe,
+ => return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ }),
+
+ .Builtin => return p.parseBuiltinCall(),
+ .Keyword_fn => return p.parseFnProto(),
+ .Keyword_if => return p.parseIf(parseTypeExpr),
+ .Keyword_switch => return p.parseSwitchExpr(),
+
+ .Keyword_extern,
+ .Keyword_packed,
+ => {
+ p.tok_i += 1;
+ return p.parseContainerDeclAuto();
+ },
- if (label) |label_token| {
- if (try p.parseBlock(label_token)) |node| return node;
- }
+ .Keyword_struct,
+ .Keyword_opaque,
+ .Keyword_enum,
+ .Keyword_union,
+ => return p.parseContainerDeclAuto(),
+
+ .Keyword_comptime => return p.addNode(.{
+ .tag = .Comptime,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.expectTypeExpr(),
+ .rhs = undefined,
+ },
+ }),
+ .MultilineStringLiteralLine => {
+ const first_line = p.nextToken();
+ while (p.token_tags[p.tok_i] == .MultilineStringLiteralLine) {
+ p.tok_i += 1;
+ }
+ return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = first_line,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
+ },
+ .Identifier => switch (p.token_tags[p.tok_i + 1]) {
+ .Colon => switch (p.token_tags[p.tok_i + 2]) {
+ .Keyword_inline => {
+ p.tok_i += 3;
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_for => return p.parseForTypeExpr(),
+ .Keyword_while => return p.parseWhileTypeExpr(),
+ else => return p.fail(.{
+ .ExpectedInlinable = .{ .token = p.tok_i },
+ }),
+ }
+ },
+ .Keyword_for => {
+ p.tok_i += 2;
+ return p.parseForTypeExpr();
+ },
+ .Keyword_while => {
+ p.tok_i += 2;
+ return p.parseWhileTypeExpr();
+ },
+ else => return p.addNode(.{
+ .tag = .Identifier,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ }),
+ },
+ else => return p.addNode(.{
+ .tag = .Identifier,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ }),
+ },
+ .Period => switch (p.token_tags[p.tok_i + 1]) {
+ .Identifier => return p.addNode(.{
+ .tag = .EnumLiteral,
+ .data = .{
+ .lhs = p.nextToken(), // dot
+ .rhs = undefined,
+ },
+ .main_token = p.nextToken(), // identifier
+ }),
+ .LBrace => {
+ const lbrace = p.tok_i + 1;
+ p.tok_i = lbrace + 1;
+
+ // If there are 0, 1, or 2 items, we can use ArrayInitDotTwo/StructInitDotTwo;
+ // otherwise we use the full ArrayInitDot/StructInitDot.
+
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .StructInitDotTwo,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = 0,
+ .rhs = 0,
+ },
+ });
+ }
+ const field_init_one = try p.parseFieldInit();
+ if (field_init_one != 0) {
+ const comma_one = p.eatToken(.Comma);
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .StructInitDotTwo,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = field_init_one,
+ .rhs = 0,
+ },
+ });
+ }
+ if (comma_one == null) {
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ }
+ const field_init_two = try p.expectFieldInit();
+ const comma_two = p.eatToken(.Comma);
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .StructInitDotTwo,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = field_init_one,
+ .rhs = field_init_two,
+ },
+ });
+ }
+ if (comma_two == null) {
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ }
+ var init_list = std.ArrayList(Node.Index).init(p.gpa);
+ defer init_list.deinit();
+
+ try init_list.appendSlice(&[_]Node.Index{ field_init_one, field_init_two });
+
+ while (true) {
+ const next = try p.expectFieldInit();
+ if (next == 0) break;
+ try init_list.append(next);
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => {
+ if (p.eatToken(.RBrace)) |_| break;
+ continue;
+ },
+ .RBrace => break,
+ .Colon, .RParen, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .RBrace,
+ },
+ });
+ },
+ else => {
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .Comma,
+ },
+ });
+ },
+ }
+ }
+ const span = try p.listToSpan(init_list.items);
+ return p.addNode(.{
+ .tag = .StructInitDot,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = span.start,
+ .rhs = span.end,
+ },
+ });
+ }
- if (try p.parseLoopTypeExpr()) |node| {
- switch (node.tag) {
- .For => node.cast(Node.For).?.label = label,
- .While => node.cast(Node.While).?.label = label,
- else => unreachable,
- }
- return node;
- }
+ const elem_init_one = try p.expectExpr();
+ const comma_one = p.eatToken(.Comma);
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .ArrayInitDotTwo,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = elem_init_one,
+ .rhs = 0,
+ },
+ });
+ }
+ if (comma_one == null) {
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ }
+ const elem_init_two = try p.expectExpr();
+ const comma_two = p.eatToken(.Comma);
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .ArrayInitDotTwo,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = elem_init_one,
+ .rhs = elem_init_two,
+ },
+ });
+ }
+ if (comma_two == null) {
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ }
+ var init_list = std.ArrayList(Node.Index).init(p.gpa);
+ defer init_list.deinit();
+
+ try init_list.appendSlice(&[_]Node.Index{ elem_init_one, elem_init_two });
+
+ while (true) {
+ const next = try p.expectExpr();
+ if (next == 0) break;
+ try init_list.append(next);
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => continue,
+ .RBrace => break,
+ .Colon, .RParen, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .RBrace,
+ },
+ });
+ },
+ else => {
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .Comma,
+ },
+ });
+ },
+ }
+ }
+ const span = try p.listToSpan(init_list.items);
+ return p.addNode(.{
+ .tag = .ArrayInitDot,
+ .main_token = lbrace,
+ .data = .{
+ .lhs = span.start,
+ .rhs = span.end,
+ },
+ });
+ },
+ else => return null_node,
+ },
+ .Keyword_error => switch (p.token_tags[p.tok_i + 1]) {
+ .LBrace => {
+ const error_token = p.tok_i;
+ p.tok_i += 2;
+
+ if (p.eatToken(.RBrace)) |_| {
+ return p.addNode(.{
+ .tag = .ErrorSetDecl,
+ .main_token = error_token,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
+ }
- if (label) |token| {
- p.putBackToken(colon);
- p.putBackToken(token);
+ while (true) {
+ const doc_comment = p.eatDocComments();
+ const identifier = try p.expectToken(.Identifier);
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => {
+ if (p.eatToken(.RBrace)) |_| break;
+ continue;
+ },
+ .RBrace => break,
+ .Colon, .RParen, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = .RBrace,
+ },
+ });
+ },
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ }
+ return p.addNode(.{
+ .tag = .ErrorSetDecl,
+ .main_token = error_token,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
+ },
+ else => return p.addNode(.{
+ .tag = .ErrorValue,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.expectToken(.Period),
+ .rhs = try p.expectToken(.Identifier),
+ },
+ }),
+ },
+ .LParen => return p.addNode(.{
+ .tag = .GroupedExpression,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = try p.expectExpr(),
+ .rhs = try p.expectToken(.RParen),
+ },
+ }),
+ else => return null_node,
}
- return null;
}
- /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr)
- fn parseLoopTypeExpr(p: *Parser) !?*Node {
- const inline_token = p.eatToken(.Keyword_inline);
-
- if (try p.parseForTypeExpr()) |node| {
- node.cast(Node.For).?.inline_token = inline_token;
- return node;
- }
-
- if (try p.parseWhileTypeExpr()) |node| {
- node.cast(Node.While).?.inline_token = inline_token;
- return node;
+ fn expectPrimaryTypeExpr(p: *Parser) !Node.Index {
+ const node = try p.parsePrimaryTypeExpr();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedPrimaryTypeExpr = .{ .token = p.tok_i } });
}
-
- if (inline_token == null) return null;
-
- // If we've seen "inline", there should have been a "for" or "while"
- try p.errors.append(p.gpa, .{
- .ExpectedInlinable = .{ .token = p.tok_i },
- });
- return error.ParseError;
+ return node;
}
+ /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
/// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)?
- fn parseForTypeExpr(p: *Parser) !?*Node {
- const node = (try p.parseForPrefix()) orelse return null;
- const for_prefix = node.cast(Node.For).?;
-
- const type_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
- });
- for_prefix.body = type_expr;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const else_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
+ fn parseForTypeExpr(p: *Parser) !Node.Index {
+ const for_token = p.eatToken(.Keyword_for) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const array_expr = try p.expectTypeExpr();
+ _ = try p.expectToken(.RParen);
+ _ = try p.parsePtrIndexPayload();
+
+ const then_expr = try p.expectExpr();
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ return p.addNode(.{
+ .tag = .ForSimple,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = then_expr,
+ },
});
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = null,
- .body = else_expr,
- };
-
- for_prefix.@"else" = else_node;
- }
-
- return node;
+ };
+ const else_expr = try p.expectTypeExpr();
+ return p.addNode(.{
+ .tag = .For,
+ .main_token = for_token,
+ .data = .{
+ .lhs = array_expr,
+ .rhs = try p.addExtra(Node.If{
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
+ });
}
+ /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
/// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)?
- fn parseWhileTypeExpr(p: *Parser) !?*Node {
- const node = (try p.parseWhilePrefix()) orelse return null;
- const while_prefix = node.cast(Node.While).?;
+ fn parseWhileTypeExpr(p: *Parser) !Node.Index {
+ const while_token = p.eatToken(.Keyword_while) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const condition = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ const then_payload = try p.parsePtrPayload();
+ const continue_expr = try p.parseWhileContinueExpr();
- const type_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
+ const then_expr = try p.expectTypeExpr();
+ const else_token = p.eatToken(.Keyword_else) orelse {
+ if (continue_expr == 0) {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileSimple else .WhileSimpleOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = then_expr,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = if (then_payload == 0) .WhileCont else .WhileContOptional,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.WhileCont{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ }),
+ },
+ });
+ }
+ };
+ const else_payload = try p.parsePayload();
+ const else_expr = try p.expectTypeExpr();
+ const tag = if (else_payload != 0)
+ Node.Tag.WhileError
+ else if (then_payload != 0)
+ Node.Tag.WhileOptional
+ else
+ Node.Tag.While;
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = while_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.While{
+ .continue_expr = continue_expr,
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
});
- while_prefix.body = type_expr;
-
- if (p.eatToken(.Keyword_else)) |else_token| {
- const payload = try p.parsePayload();
-
- const else_expr = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
- });
-
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = null,
- .body = else_expr,
- };
-
- while_prefix.@"else" = else_node;
- }
-
- return node;
}
/// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE
- fn parseSwitchExpr(p: *Parser) !?*Node {
- const switch_token = p.eatToken(.Keyword_switch) orelse return null;
+ fn parseSwitchExpr(p: *Parser) !Node.Index {
+ const switch_token = p.eatToken(.Keyword_switch) orelse return null_node;
_ = try p.expectToken(.LParen);
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ const expr_node = try p.expectExpr();
_ = try p.expectToken(.RParen);
_ = try p.expectToken(.LBrace);
const cases = try p.parseSwitchProngList();
- defer p.gpa.free(cases);
- const rbrace = try p.expectToken(.RBrace);
-
- const node = try Node.Switch.alloc(&p.arena.allocator, cases.len);
- node.* = .{
- .switch_token = switch_token,
- .expr = expr_node,
- .cases_len = cases.len,
- .rbrace = rbrace,
- };
- std.mem.copy(*Node, node.cases(), cases);
- return &node.base;
+ _ = try p.expectToken(.RBrace);
+
+ return p.addNode(.{
+ .tag = .Switch,
+ .main_token = switch_token,
+ .data = .{
+ .lhs = expr_node,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = cases.start,
+ .end = cases.end,
+ }),
+ },
+ });
}
/// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN
@@ -1800,1696 +2713,939 @@ const Parser = struct {
/// AsmInput <- COLON AsmInputList AsmClobbers?
/// AsmClobbers <- COLON StringList
/// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL?
- fn parseAsmExpr(p: *Parser) !?*Node {
- const asm_token = p.eatToken(.Keyword_asm) orelse return null;
- const volatile_token = p.eatToken(.Keyword_volatile);
+ /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
+ /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
+ fn parseAsmExpr(p: *Parser) !Node.Index {
+ const asm_token = p.assertToken(.Keyword_asm);
+ _ = p.eatToken(.Keyword_volatile);
_ = try p.expectToken(.LParen);
- const template = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- var arena_outputs: []Node.Asm.Output = &[0]Node.Asm.Output{};
- var arena_inputs: []Node.Asm.Input = &[0]Node.Asm.Input{};
- var arena_clobbers: []*Node = &[0]*Node{};
-
- if (p.eatToken(.Colon) != null) {
- const outputs = try p.parseAsmOutputList();
- defer p.gpa.free(outputs);
- arena_outputs = try p.arena.allocator.dupe(Node.Asm.Output, outputs);
-
- if (p.eatToken(.Colon) != null) {
- const inputs = try p.parseAsmInputList();
- defer p.gpa.free(inputs);
- arena_inputs = try p.arena.allocator.dupe(Node.Asm.Input, inputs);
-
- if (p.eatToken(.Colon) != null) {
- const clobbers = try ListParseFn(*Node, parseStringLiteral)(p);
- defer p.gpa.free(clobbers);
- arena_clobbers = try p.arena.allocator.dupe(*Node, clobbers);
- }
- }
+ const template = try p.expectExpr();
+
+ if (p.eatToken(.RParen)) |_| {
+ return p.addNode(.{
+ .tag = .AsmSimple,
+ .main_token = asm_token,
+ .data = .{
+ .lhs = template,
+ .rhs = undefined,
+ },
+ });
}
- const node = try p.arena.allocator.create(Node.Asm);
- node.* = .{
- .asm_token = asm_token,
- .volatile_token = volatile_token,
- .template = template,
- .outputs = arena_outputs,
- .inputs = arena_inputs,
- .clobbers = arena_clobbers,
- .rparen = try p.expectToken(.RParen),
- };
-
- return &node.base;
- }
+ _ = try p.expectToken(.Colon);
- /// DOT IDENTIFIER
- fn parseAnonLiteral(p: *Parser) !?*Node {
- const dot = p.eatToken(.Period) orelse return null;
+ var list = std.ArrayList(Node.Index).init(p.gpa);
+ defer list.deinit();
- // anon enum literal
- if (p.eatToken(.Identifier)) |name| {
- const node = try p.arena.allocator.create(Node.EnumLiteral);
- node.* = .{
- .dot = dot,
- .name = name,
- };
- return &node.base;
+ while (true) {
+ const output_item = try p.parseAsmOutputItem();
+ if (output_item == 0) break;
+ try list.append(output_item);
+ switch (p.token_tags[p.tok_i]) {
+ .Comma => p.tok_i += 1,
+ .Colon, .RParen, .RBrace, .RBracket => break, // All possible delimiters.
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
}
-
- if (try p.parseAnonInitList(dot)) |node| {
- return node;
+ if (p.eatToken(.Colon)) |_| {
+ while (true) {
+ const input_item = try p.parseAsmInputItem();
+ if (input_item == 0) break;
+ try list.append(input_item);
+ switch (p.token_tags[p.tok_i]) {
+ .Comma => p.tok_i += 1,
+ .Colon, .RParen, .RBrace, .RBracket => break, // All possible delimiters.
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ }
+ if (p.eatToken(.Colon)) |_| {
+ while (p.eatToken(.StringLiteral)) |_| {
+ switch (p.token_tags[p.tok_i]) {
+ .Comma => p.tok_i += 1,
+ .Colon, .RParen, .RBrace, .RBracket => break,
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ }
+ }
}
-
- p.putBackToken(dot);
- return null;
+ _ = try p.expectToken(.RParen);
+ const span = try p.listToSpan(list.items);
+ return p.addNode(.{
+ .tag = .Asm,
+ .main_token = asm_token,
+ .data = .{
+ .lhs = template,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = span.start,
+ .end = span.end,
+ }),
+ },
+ });
}
/// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN
- fn parseAsmOutputItem(p: *Parser) !?Node.Asm.Output {
- const lbracket = p.eatToken(.LBracket) orelse return null;
- const name = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
+ fn parseAsmOutputItem(p: *Parser) !Node.Index {
+ _ = p.eatToken(.LBracket) orelse return null_node;
+ const identifier = try p.expectToken(.Identifier);
_ = try p.expectToken(.RBracket);
-
- const constraint = try p.expectNode(parseStringLiteral, .{
- .ExpectedStringLiteral = .{ .token = p.tok_i },
- });
-
+ const constraint = try p.expectToken(.StringLiteral);
_ = try p.expectToken(.LParen);
- const kind: Node.Asm.Output.Kind = blk: {
- if (p.eatToken(.Arrow) != null) {
- const return_ident = try p.expectNode(parseTypeExpr, .{
- .ExpectedTypeExpr = .{ .token = p.tok_i },
- });
- break :blk .{ .Return = return_ident };
- }
- const variable = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
- break :blk .{ .Variable = variable.castTag(.Identifier).? };
- };
- const rparen = try p.expectToken(.RParen);
-
- return Node.Asm.Output{
- .lbracket = lbracket,
- .symbolic_name = name,
- .constraint = constraint,
- .kind = kind,
- .rparen = rparen,
- };
+ const rhs: Node.Index = if (p.eatToken(.Arrow)) |_| try p.expectTypeExpr() else null_node;
+ _ = try p.expectToken(.RParen);
+ return p.addNode(.{
+ .tag = .AsmOutput,
+ .main_token = identifier,
+ .data = .{
+ .lhs = constraint,
+ .rhs = rhs,
+ },
+ });
}
/// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN
- fn parseAsmInputItem(p: *Parser) !?Node.Asm.Input {
- const lbracket = p.eatToken(.LBracket) orelse return null;
- const name = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
+ fn parseAsmInputItem(p: *Parser) !Node.Index {
+ _ = p.eatToken(.LBracket) orelse return null_node;
+ const identifier = try p.expectToken(.Identifier);
_ = try p.expectToken(.RBracket);
-
- const constraint = try p.expectNode(parseStringLiteral, .{
- .ExpectedStringLiteral = .{ .token = p.tok_i },
- });
-
+ const constraint = try p.expectToken(.StringLiteral);
_ = try p.expectToken(.LParen);
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
+ const expr = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ return p.addNode(.{
+ .tag = .AsmInput,
+ .main_token = identifier,
+ .data = .{
+ .lhs = constraint,
+ .rhs = expr,
+ },
});
- const rparen = try p.expectToken(.RParen);
-
- return Node.Asm.Input{
- .lbracket = lbracket,
- .symbolic_name = name,
- .constraint = constraint,
- .expr = expr,
- .rparen = rparen,
- };
}
/// BreakLabel <- COLON IDENTIFIER
- fn parseBreakLabel(p: *Parser) !?TokenIndex {
- _ = p.eatToken(.Colon) orelse return null;
- const ident = try p.expectToken(.Identifier);
- return ident;
+ fn parseBreakLabel(p: *Parser) !TokenIndex {
+ _ = p.eatToken(.Colon) orelse return @as(TokenIndex, 0);
+ return p.expectToken(.Identifier);
}
/// BlockLabel <- IDENTIFIER COLON
- fn parseBlockLabel(p: *Parser, colon_token: *TokenIndex) ?TokenIndex {
- const identifier = p.eatToken(.Identifier) orelse return null;
- if (p.eatToken(.Colon)) |colon| {
- colon_token.* = colon;
+ fn parseBlockLabel(p: *Parser) TokenIndex {
+ if (p.token_tags[p.tok_i] == .Identifier and
+ p.token_tags[p.tok_i + 1] == .Colon)
+ {
+ const identifier = p.tok_i;
+ p.tok_i += 2;
return identifier;
}
- p.putBackToken(identifier);
- return null;
+ return 0;
}
/// FieldInit <- DOT IDENTIFIER EQUAL Expr
- fn parseFieldInit(p: *Parser) !?*Node {
- const period_token = p.eatToken(.Period) orelse return null;
- const name_token = p.eatToken(.Identifier) orelse {
- // Because of anon literals `.{` is also valid.
- p.putBackToken(period_token);
- return null;
- };
- const eq_token = p.eatToken(.Equal) orelse {
- // `.Name` may also be an enum literal, which is a later rule.
- p.putBackToken(name_token);
- p.putBackToken(period_token);
- return null;
- };
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ fn parseFieldInit(p: *Parser) !Node.Index {
+ if (p.token_tags[p.tok_i + 0] == .Period and
+ p.token_tags[p.tok_i + 1] == .Identifier and
+ p.token_tags[p.tok_i + 2] == .Equal)
+ {
+ p.tok_i += 3;
+ return p.expectExpr();
+ } else {
+ return null_node;
+ }
+ }
- const node = try p.arena.allocator.create(Node.FieldInitializer);
- node.* = .{
- .period_token = period_token,
- .name_token = name_token,
- .expr = expr_node,
- };
- return &node.base;
+ fn expectFieldInit(p: *Parser) !Node.Index {
+ _ = try p.expectToken(.Period);
+ _ = try p.expectToken(.Identifier);
+ _ = try p.expectToken(.Equal);
+ return p.expectExpr();
}
/// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN
- fn parseWhileContinueExpr(p: *Parser) !?*Node {
- _ = p.eatToken(.Colon) orelse return null;
+ fn parseWhileContinueExpr(p: *Parser) !Node.Index {
+ _ = p.eatToken(.Colon) orelse return null_node;
_ = try p.expectToken(.LParen);
- const node = try p.expectNode(parseAssignExpr, .{
- .ExpectedExprOrAssignment = .{ .token = p.tok_i },
- });
+ const node = try p.parseAssignExpr();
+ if (node == 0) return p.fail(.{ .ExpectedExprOrAssignment = .{ .token = p.tok_i } });
_ = try p.expectToken(.RParen);
return node;
}
/// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN
- fn parseLinkSection(p: *Parser) !?*Node {
- _ = p.eatToken(.Keyword_linksection) orelse return null;
+ fn parseLinkSection(p: *Parser) !Node.Index {
+ _ = p.eatToken(.Keyword_linksection) orelse return null_node;
_ = try p.expectToken(.LParen);
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ const expr_node = try p.expectExpr();
_ = try p.expectToken(.RParen);
return expr_node;
}
/// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN
- fn parseCallconv(p: *Parser) !?*Node {
- _ = p.eatToken(.Keyword_callconv) orelse return null;
+ fn parseCallconv(p: *Parser) !Node.Index {
+ _ = p.eatToken(.Keyword_callconv) orelse return null_node;
_ = try p.expectToken(.LParen);
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ const expr_node = try p.expectExpr();
_ = try p.expectToken(.RParen);
return expr_node;
}
- /// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
- fn parseParamDecl(p: *Parser) !?Node.FnProto.ParamDecl {
- const doc_comments = try p.parseDocComment();
- const noalias_token = p.eatToken(.Keyword_noalias);
- const comptime_token = if (noalias_token == null) p.eatToken(.Keyword_comptime) else null;
- const name_token = blk: {
- const identifier = p.eatToken(.Identifier) orelse break :blk null;
- if (p.eatToken(.Colon) != null) break :blk identifier;
- p.putBackToken(identifier); // ParamType may also be an identifier
- break :blk null;
- };
- const param_type = (try p.parseParamType()) orelse {
- // Only return cleanly if no keyword, identifier, or doc comment was found
- if (noalias_token == null and
- comptime_token == null and
- name_token == null and
- doc_comments == null)
- {
- return null;
- }
- try p.errors.append(p.gpa, .{
- .ExpectedParamType = .{ .token = p.tok_i },
- });
- return error.ParseError;
- };
-
- return Node.FnProto.ParamDecl{
- .doc_comments = doc_comments,
- .comptime_token = comptime_token,
- .noalias_token = noalias_token,
- .name_token = name_token,
- .param_type = param_type,
- };
- }
-
+ /// ParamDecl
+ /// <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
+ /// / DOT3
/// ParamType
/// <- Keyword_anytype
- /// / DOT3
/// / TypeExpr
- fn parseParamType(p: *Parser) !?Node.FnProto.ParamDecl.ParamType {
- // TODO cast from tuple to error union is broken
- const P = Node.FnProto.ParamDecl.ParamType;
- if (try p.parseAnyType()) |node| return P{ .any_type = node };
- if (try p.parseTypeExpr()) |node| return P{ .type_expr = node };
- return null;
+ /// This function can return null nodes and then still return nodes afterwards,
+ /// such as in the case of anytype and `...`. Caller must look for rparen to find
+ /// out when there are no more param decls left.
+ fn expectParamDecl(p: *Parser) !Node.Index {
+ _ = p.eatDocComments();
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_noalias, .Keyword_comptime => p.tok_i += 1,
+ .Ellipsis3 => {
+ p.tok_i += 1;
+ return null_node;
+ },
+ else => {},
+ }
+ if (p.token_tags[p.tok_i] == .Identifier and
+ p.token_tags[p.tok_i + 1] == .Colon)
+ {
+ p.tok_i += 2;
+ }
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_anytype => {
+ p.tok_i += 1;
+ return null_node;
+ },
+ else => return p.expectTypeExpr(),
+ }
}
- /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
- fn parseIfPrefix(p: *Parser) !?*Node {
- const if_token = p.eatToken(.Keyword_if) orelse return null;
- _ = try p.expectToken(.LParen);
- const condition = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- _ = try p.expectToken(.RParen);
- const payload = try p.parsePtrPayload();
-
- const node = try p.arena.allocator.create(Node.If);
- node.* = .{
- .if_token = if_token,
- .condition = condition,
- .payload = payload,
- .body = undefined, // set by caller
- .@"else" = null,
- };
- return &node.base;
+ /// Payload <- PIPE IDENTIFIER PIPE
+ fn parsePayload(p: *Parser) !TokenIndex {
+ _ = p.eatToken(.Pipe) orelse return @as(TokenIndex, 0);
+ const identifier = try p.expectToken(.Identifier);
+ _ = try p.expectToken(.Pipe);
+ return identifier;
}
- /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr?
- fn parseWhilePrefix(p: *Parser) !?*Node {
- const while_token = p.eatToken(.Keyword_while) orelse return null;
+ /// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
+ fn parsePtrPayload(p: *Parser) !TokenIndex {
+ _ = p.eatToken(.Pipe) orelse return @as(TokenIndex, 0);
+ _ = p.eatToken(.Asterisk);
+ const identifier = try p.expectToken(.Identifier);
+ _ = try p.expectToken(.Pipe);
+ return identifier;
+ }
- _ = try p.expectToken(.LParen);
- const condition = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- _ = try p.expectToken(.RParen);
+ /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
+ /// Returns the first identifier token, if any.
+ fn parsePtrIndexPayload(p: *Parser) !TokenIndex {
+ _ = p.eatToken(.Pipe) orelse return @as(TokenIndex, 0);
+ _ = p.eatToken(.Asterisk);
+ const identifier = try p.expectToken(.Identifier);
+ if (p.eatToken(.Comma) != null) {
+ _ = try p.expectToken(.Identifier);
+ }
+ _ = try p.expectToken(.Pipe);
+ return identifier;
+ }
- const payload = try p.parsePtrPayload();
- const continue_expr = try p.parseWhileContinueExpr();
+ /// SwitchProng <- SwitchCase EQUALRARROW PtrPayload? AssignExpr
+ /// SwitchCase
+ /// <- SwitchItem (COMMA SwitchItem)* COMMA?
+ /// / KEYWORD_else
+ fn parseSwitchProng(p: *Parser) !Node.Index {
+ if (p.eatToken(.Keyword_else)) |_| {
+ const arrow_token = try p.expectToken(.EqualAngleBracketRight);
+ _ = try p.parsePtrPayload();
+ return p.addNode(.{
+ .tag = .SwitchCaseOne,
+ .main_token = arrow_token,
+ .data = .{
+ .lhs = 0,
+ .rhs = try p.expectAssignExpr(),
+ },
+ });
+ }
+ const first_item = try p.parseSwitchItem();
+ if (first_item == 0) return null_node;
+
+ if (p.token_tags[p.tok_i] == .RBrace) {
+ const arrow_token = try p.expectToken(.EqualAngleBracketRight);
+ _ = try p.parsePtrPayload();
+ return p.addNode(.{
+ .tag = .SwitchCaseOne,
+ .main_token = arrow_token,
+ .data = .{
+ .lhs = first_item,
+ .rhs = try p.expectAssignExpr(),
+ },
+ });
+ }
- const node = try p.arena.allocator.create(Node.While);
- node.* = .{
- .label = null,
- .inline_token = null,
- .while_token = while_token,
- .condition = condition,
- .payload = payload,
- .continue_expr = continue_expr,
- .body = undefined, // set by caller
- .@"else" = null,
- };
- return &node.base;
- }
-
- /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload
- fn parseForPrefix(p: *Parser) !?*Node {
- const for_token = p.eatToken(.Keyword_for) orelse return null;
-
- _ = try p.expectToken(.LParen);
- const array_expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- _ = try p.expectToken(.RParen);
-
- const payload = try p.expectNode(parsePtrIndexPayload, .{
- .ExpectedPayload = .{ .token = p.tok_i },
- });
-
- const node = try p.arena.allocator.create(Node.For);
- node.* = .{
- .label = null,
- .inline_token = null,
- .for_token = for_token,
- .array_expr = array_expr,
- .payload = payload,
- .body = undefined, // set by caller
- .@"else" = null,
- };
- return &node.base;
- }
-
- /// Payload <- PIPE IDENTIFIER PIPE
- fn parsePayload(p: *Parser) !?*Node {
- const lpipe = p.eatToken(.Pipe) orelse return null;
- const identifier = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
- const rpipe = try p.expectToken(.Pipe);
-
- const node = try p.arena.allocator.create(Node.Payload);
- node.* = .{
- .lpipe = lpipe,
- .error_symbol = identifier,
- .rpipe = rpipe,
- };
- return &node.base;
- }
-
- /// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE
- fn parsePtrPayload(p: *Parser) !?*Node {
- const lpipe = p.eatToken(.Pipe) orelse return null;
- const asterisk = p.eatToken(.Asterisk);
- const identifier = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
- const rpipe = try p.expectToken(.Pipe);
-
- const node = try p.arena.allocator.create(Node.PointerPayload);
- node.* = .{
- .lpipe = lpipe,
- .ptr_token = asterisk,
- .value_symbol = identifier,
- .rpipe = rpipe,
- };
- return &node.base;
- }
-
- /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE
- fn parsePtrIndexPayload(p: *Parser) !?*Node {
- const lpipe = p.eatToken(.Pipe) orelse return null;
- const asterisk = p.eatToken(.Asterisk);
- const identifier = try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
-
- const index = if (p.eatToken(.Comma) == null)
- null
- else
- try p.expectNode(parseIdentifier, .{
- .ExpectedIdentifier = .{ .token = p.tok_i },
- });
-
- const rpipe = try p.expectToken(.Pipe);
-
- const node = try p.arena.allocator.create(Node.PointerIndexPayload);
- node.* = .{
- .lpipe = lpipe,
- .ptr_token = asterisk,
- .value_symbol = identifier,
- .index_symbol = index,
- .rpipe = rpipe,
- };
- return &node.base;
- }
-
- /// SwitchProng <- SwitchCase EQUALRARROW PtrPayload? AssignExpr
- fn parseSwitchProng(p: *Parser) !?*Node {
- const node = (try p.parseSwitchCase()) orelse return null;
- const arrow = try p.expectToken(.EqualAngleBracketRight);
- const payload = try p.parsePtrPayload();
- const expr = try p.expectNode(parseAssignExpr, .{
- .ExpectedExprOrAssignment = .{ .token = p.tok_i },
- });
-
- const switch_case = node.cast(Node.SwitchCase).?;
- switch_case.arrow_token = arrow;
- switch_case.payload = payload;
- switch_case.expr = expr;
-
- return node;
- }
-
- /// SwitchCase
- /// <- SwitchItem (COMMA SwitchItem)* COMMA?
- /// / KEYWORD_else
- fn parseSwitchCase(p: *Parser) !?*Node {
- var list = std.ArrayList(*Node).init(p.gpa);
- defer list.deinit();
-
- if (try p.parseSwitchItem()) |first_item| {
- try list.append(first_item);
- while (p.eatToken(.Comma) != null) {
- const next_item = (try p.parseSwitchItem()) orelse break;
- try list.append(next_item);
- }
- } else if (p.eatToken(.Keyword_else)) |else_token| {
- const else_node = try p.arena.allocator.create(Node.SwitchElse);
- else_node.* = .{
- .token = else_token,
- };
- try list.append(&else_node.base);
- } else return null;
-
- const node = try Node.SwitchCase.alloc(&p.arena.allocator, list.items.len);
- node.* = .{
- .items_len = list.items.len,
- .arrow_token = undefined, // set by caller
- .payload = null,
- .expr = undefined, // set by caller
- };
- std.mem.copy(*Node, node.items(), list.items);
- return &node.base;
+ var list = std.ArrayList(Node.Index).init(p.gpa);
+ defer list.deinit();
+
+ try list.append(first_item);
+ while (p.eatToken(.Comma)) |_| {
+ const next_item = try p.parseSwitchItem();
+ if (next_item == 0) break;
+ try list.append(next_item);
+ }
+ const span = try p.listToSpan(list.items);
+ const arrow_token = try p.expectToken(.EqualAngleBracketRight);
+ _ = try p.parsePtrPayload();
+ return p.addNode(.{
+ .tag = .SwitchCaseMulti,
+ .main_token = arrow_token,
+ .data = .{
+ .lhs = try p.addExtra(Node.SubRange{
+ .start = span.start,
+ .end = span.end,
+ }),
+ .rhs = try p.expectAssignExpr(),
+ },
+ });
}
/// SwitchItem <- Expr (DOT3 Expr)?
- fn parseSwitchItem(p: *Parser) !?*Node {
- const expr = (try p.parseExpr()) orelse return null;
- if (p.eatToken(.Ellipsis3)) |token| {
- const range_end = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = Node{ .tag = .Range },
- .op_token = token,
- .lhs = expr,
- .rhs = range_end,
- };
- return &node.base;
- }
- return expr;
- }
-
- /// AssignOp
- /// <- ASTERISKEQUAL
- /// / SLASHEQUAL
- /// / PERCENTEQUAL
- /// / PLUSEQUAL
- /// / MINUSEQUAL
- /// / LARROW2EQUAL
- /// / RARROW2EQUAL
- /// / AMPERSANDEQUAL
- /// / CARETEQUAL
- /// / PIPEEQUAL
- /// / ASTERISKPERCENTEQUAL
- /// / PLUSPERCENTEQUAL
- /// / MINUSPERCENTEQUAL
- /// / EQUAL
- fn parseAssignOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .AsteriskEqual => .AssignMul,
- .SlashEqual => .AssignDiv,
- .PercentEqual => .AssignMod,
- .PlusEqual => .AssignAdd,
- .MinusEqual => .AssignSub,
- .AngleBracketAngleBracketLeftEqual => .AssignBitShiftLeft,
- .AngleBracketAngleBracketRightEqual => .AssignBitShiftRight,
- .AmpersandEqual => .AssignBitAnd,
- .CaretEqual => .AssignBitXor,
- .PipeEqual => .AssignBitOr,
- .AsteriskPercentEqual => .AssignMulWrap,
- .PlusPercentEqual => .AssignAddWrap,
- .MinusPercentEqual => .AssignSubWrap,
- .Equal => .Assign,
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = .{ .tag = op },
- .op_token = token,
- .lhs = undefined, // set by caller
- .rhs = undefined, // set by caller
- };
- return &node.base;
- }
-
- /// CompareOp
- /// <- EQUALEQUAL
- /// / EXCLAMATIONMARKEQUAL
- /// / LARROW
- /// / RARROW
- /// / LARROWEQUAL
- /// / RARROWEQUAL
- fn parseCompareOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .EqualEqual => .EqualEqual,
- .BangEqual => .BangEqual,
- .AngleBracketLeft => .LessThan,
- .AngleBracketRight => .GreaterThan,
- .AngleBracketLeftEqual => .LessOrEqual,
- .AngleBracketRightEqual => .GreaterOrEqual,
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- return p.createInfixOp(token, op);
- }
-
- /// BitwiseOp
- /// <- AMPERSAND
- /// / CARET
- /// / PIPE
- /// / KEYWORD_orelse
- /// / KEYWORD_catch Payload?
- fn parseBitwiseOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .Ampersand => .BitAnd,
- .Caret => .BitXor,
- .Pipe => .BitOr,
- .Keyword_orelse => .OrElse,
- .Keyword_catch => {
- const payload = try p.parsePayload();
- const node = try p.arena.allocator.create(Node.Catch);
- node.* = .{
- .op_token = token,
- .lhs = undefined, // set by caller
- .rhs = undefined, // set by caller
- .payload = payload,
- };
- return &node.base;
- },
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- return p.createInfixOp(token, op);
- }
-
- /// BitShiftOp
- /// <- LARROW2
- /// / RARROW2
- fn parseBitShiftOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .AngleBracketAngleBracketLeft => .BitShiftLeft,
- .AngleBracketAngleBracketRight => .BitShiftRight,
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- return p.createInfixOp(token, op);
- }
-
- /// AdditionOp
- /// <- PLUS
- /// / MINUS
- /// / PLUS2
- /// / PLUSPERCENT
- /// / MINUSPERCENT
- fn parseAdditionOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .Plus => .Add,
- .Minus => .Sub,
- .PlusPlus => .ArrayCat,
- .PlusPercent => .AddWrap,
- .MinusPercent => .SubWrap,
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- return p.createInfixOp(token, op);
- }
-
- /// MultiplyOp
- /// <- PIPE2
- /// / ASTERISK
- /// / SLASH
- /// / PERCENT
- /// / ASTERISK2
- /// / ASTERISKPERCENT
- fn parseMultiplyOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- const op: Node.Tag = switch (p.token_ids[token]) {
- .PipePipe => .MergeErrorSets,
- .Asterisk => .Mul,
- .Slash => .Div,
- .Percent => .Mod,
- .AsteriskAsterisk => .ArrayMult,
- .AsteriskPercent => .MulWrap,
- else => {
- p.putBackToken(token);
- return null;
- },
- };
-
- return p.createInfixOp(token, op);
- }
-
- /// PrefixOp
- /// <- EXCLAMATIONMARK
- /// / MINUS
- /// / TILDE
- /// / MINUSPERCENT
- /// / AMPERSAND
- /// / KEYWORD_try
- /// / KEYWORD_await
- fn parsePrefixOp(p: *Parser) !?*Node {
- const token = p.nextToken();
- switch (p.token_ids[token]) {
- .Bang => return p.allocSimplePrefixOp(.BoolNot, token),
- .Minus => return p.allocSimplePrefixOp(.Negation, token),
- .Tilde => return p.allocSimplePrefixOp(.BitNot, token),
- .MinusPercent => return p.allocSimplePrefixOp(.NegationWrap, token),
- .Ampersand => return p.allocSimplePrefixOp(.AddressOf, token),
- .Keyword_try => return p.allocSimplePrefixOp(.Try, token),
- .Keyword_await => return p.allocSimplePrefixOp(.Await, token),
- else => {
- p.putBackToken(token);
- return null;
- },
- }
- }
-
- fn allocSimplePrefixOp(p: *Parser, comptime tag: Node.Tag, token: TokenIndex) !?*Node {
- const node = try p.arena.allocator.create(Node.SimplePrefixOp);
- node.* = .{
- .base = .{ .tag = tag },
- .op_token = token,
- .rhs = undefined, // set by caller
- };
- return &node.base;
- }
-
- // TODO: ArrayTypeStart is either an array or a slice, but const/allowzero only work on
- // pointers. Consider updating this rule:
- // ...
- // / ArrayTypeStart
- // / SliceTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
- // / PtrTypeStart ...
-
- /// PrefixTypeOp
- /// <- QUESTIONMARK
- /// / KEYWORD_anyframe MINUSRARROW
- /// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
- /// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
- fn parsePrefixTypeOp(p: *Parser) !?*Node {
- if (p.eatToken(.QuestionMark)) |token| {
- const node = try p.arena.allocator.create(Node.SimplePrefixOp);
- node.* = .{
- .base = .{ .tag = .OptionalType },
- .op_token = token,
- .rhs = undefined, // set by caller
- };
- return &node.base;
- }
+ fn parseSwitchItem(p: *Parser) !Node.Index {
+ const expr = try p.parseExpr();
+ if (expr == 0) return null_node;
- if (p.eatToken(.Keyword_anyframe)) |token| {
- const arrow = p.eatToken(.Arrow) orelse {
- p.putBackToken(token);
- return null;
- };
- const node = try p.arena.allocator.create(Node.AnyFrameType);
- node.* = .{
- .anyframe_token = token,
- .result = .{
- .arrow_token = arrow,
- .return_type = undefined, // set by caller
+ if (p.eatToken(.Ellipsis3)) |token| {
+ return p.addNode(.{
+ .tag = .SwitchRange,
+ .main_token = token,
+ .data = .{
+ .lhs = expr,
+ .rhs = try p.expectExpr(),
},
- };
- return &node.base;
- }
-
- if (try p.parsePtrTypeStart()) |node| {
- // If the token encountered was **, there will be two nodes instead of one.
- // The attributes should be applied to the rightmost operator.
- var ptr_info = if (node.cast(Node.PtrType)) |ptr_type|
- if (p.token_ids[ptr_type.op_token] == .AsteriskAsterisk)
- &ptr_type.rhs.cast(Node.PtrType).?.ptr_info
- else
- &ptr_type.ptr_info
- else if (node.cast(Node.SliceType)) |slice_type|
- &slice_type.ptr_info
- else
- unreachable;
-
- while (true) {
- if (p.eatToken(.Keyword_align)) |align_token| {
- const lparen = try p.expectToken(.LParen);
- const expr_node = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- // Optional bit range
- const bit_range = if (p.eatToken(.Colon)) |_| bit_range_value: {
- const range_start = try p.expectNode(parseIntegerLiteral, .{
- .ExpectedIntegerLiteral = .{ .token = p.tok_i },
- });
- _ = try p.expectToken(.Colon);
- const range_end = try p.expectNode(parseIntegerLiteral, .{
- .ExpectedIntegerLiteral = .{ .token = p.tok_i },
- });
-
- break :bit_range_value ast.PtrInfo.Align.BitRange{
- .start = range_start,
- .end = range_end,
- };
- } else null;
- _ = try p.expectToken(.RParen);
-
- if (ptr_info.align_info != null) {
- try p.errors.append(p.gpa, .{
- .ExtraAlignQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
-
- ptr_info.align_info = ast.PtrInfo.Align{
- .node = expr_node,
- .bit_range = bit_range,
- };
-
- continue;
- }
- if (p.eatToken(.Keyword_const)) |const_token| {
- if (ptr_info.const_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraConstQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- ptr_info.const_token = const_token;
- continue;
- }
- if (p.eatToken(.Keyword_volatile)) |volatile_token| {
- if (ptr_info.volatile_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraVolatileQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- ptr_info.volatile_token = volatile_token;
- continue;
- }
- if (p.eatToken(.Keyword_allowzero)) |allowzero_token| {
- if (ptr_info.allowzero_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraAllowZeroQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- ptr_info.allowzero_token = allowzero_token;
- continue;
- }
- break;
- }
-
- return node;
- }
-
- if (try p.parseArrayTypeStart()) |node| {
- if (node.cast(Node.SliceType)) |slice_type| {
- // Collect pointer qualifiers in any order, but disallow duplicates
- while (true) {
- if (try p.parseByteAlign()) |align_expr| {
- if (slice_type.ptr_info.align_info != null) {
- try p.errors.append(p.gpa, .{
- .ExtraAlignQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- slice_type.ptr_info.align_info = ast.PtrInfo.Align{
- .node = align_expr,
- .bit_range = null,
- };
- continue;
- }
- if (p.eatToken(.Keyword_const)) |const_token| {
- if (slice_type.ptr_info.const_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraConstQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- slice_type.ptr_info.const_token = const_token;
- continue;
- }
- if (p.eatToken(.Keyword_volatile)) |volatile_token| {
- if (slice_type.ptr_info.volatile_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraVolatileQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- slice_type.ptr_info.volatile_token = volatile_token;
- continue;
- }
- if (p.eatToken(.Keyword_allowzero)) |allowzero_token| {
- if (slice_type.ptr_info.allowzero_token != null) {
- try p.errors.append(p.gpa, .{
- .ExtraAllowZeroQualifier = .{ .token = p.tok_i - 1 },
- });
- continue;
- }
- slice_type.ptr_info.allowzero_token = allowzero_token;
- continue;
- }
- break;
- }
- }
- return node;
- }
-
- return null;
- }
-
- /// SuffixOp
- /// <- LBRACKET Expr (DOT2 (Expr (COLON Expr)?)?)? RBRACKET
- /// / DOT IDENTIFIER
- /// / DOTASTERISK
- /// / DOTQUESTIONMARK
- fn parseSuffixOp(p: *Parser, lhs: *Node) !?*Node {
- if (p.eatToken(.LBracket)) |_| {
- const index_expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
-
- if (p.eatToken(.Ellipsis2) != null) {
- const end_expr = try p.parseExpr();
- const sentinel: ?*Node = if (p.eatToken(.Colon) != null)
- try p.parseExpr()
- else
- null;
- const rtoken = try p.expectToken(.RBracket);
- const node = try p.arena.allocator.create(Node.Slice);
- node.* = .{
- .lhs = lhs,
- .rtoken = rtoken,
- .start = index_expr,
- .end = end_expr,
- .sentinel = sentinel,
- };
- return &node.base;
- }
-
- const rtoken = try p.expectToken(.RBracket);
- const node = try p.arena.allocator.create(Node.ArrayAccess);
- node.* = .{
- .lhs = lhs,
- .rtoken = rtoken,
- .index_expr = index_expr,
- };
- return &node.base;
- }
-
- if (p.eatToken(.PeriodAsterisk)) |period_asterisk| {
- const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
- node.* = .{
- .base = .{ .tag = .Deref },
- .lhs = lhs,
- .rtoken = period_asterisk,
- };
- return &node.base;
- }
-
- if (p.eatToken(.Invalid_periodasterisks)) |period_asterisk| {
- try p.errors.append(p.gpa, .{
- .AsteriskAfterPointerDereference = .{ .token = period_asterisk },
- });
- const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
- node.* = .{
- .base = .{ .tag = .Deref },
- .lhs = lhs,
- .rtoken = period_asterisk,
- };
- return &node.base;
- }
-
- if (p.eatToken(.Period)) |period| {
- if (try p.parseIdentifier()) |identifier| {
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = Node{ .tag = .Period },
- .op_token = period,
- .lhs = lhs,
- .rhs = identifier,
- };
- return &node.base;
- }
- if (p.eatToken(.QuestionMark)) |question_mark| {
- const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
- node.* = .{
- .base = .{ .tag = .UnwrapOptional },
- .lhs = lhs,
- .rtoken = question_mark,
- };
- return &node.base;
- }
- try p.errors.append(p.gpa, .{
- .ExpectedSuffixOp = .{ .token = p.tok_i },
});
- return null;
}
-
- return null;
- }
-
- /// FnCallArguments <- LPAREN ExprList RPAREN
- /// ExprList <- (Expr COMMA)* Expr?
- fn parseFnCallArguments(p: *Parser) !?AnnotatedParamList {
- if (p.eatToken(.LParen) == null) return null;
- const list = try ListParseFn(*Node, parseExpr)(p);
- errdefer p.gpa.free(list);
- const rparen = try p.expectToken(.RParen);
- return AnnotatedParamList{ .list = list, .rparen = rparen };
- }
-
- const AnnotatedParamList = struct {
- list: []*Node,
- rparen: TokenIndex,
- };
-
- /// ArrayTypeStart <- LBRACKET Expr? (COLON Expr)? RBRACKET
- fn parseArrayTypeStart(p: *Parser) !?*Node {
- const lbracket = p.eatToken(.LBracket) orelse return null;
- const expr = try p.parseExpr();
- const sentinel = if (p.eatToken(.Colon)) |_|
- try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- })
- else
- null;
- const rbracket = try p.expectToken(.RBracket);
-
- if (expr) |len_expr| {
- if (sentinel) |s| {
- const node = try p.arena.allocator.create(Node.ArrayTypeSentinel);
- node.* = .{
- .op_token = lbracket,
- .rhs = undefined, // set by caller
- .len_expr = len_expr,
- .sentinel = s,
- };
- return &node.base;
- } else {
- const node = try p.arena.allocator.create(Node.ArrayType);
- node.* = .{
- .op_token = lbracket,
- .rhs = undefined, // set by caller
- .len_expr = len_expr,
- };
- return &node.base;
- }
- }
-
- const node = try p.arena.allocator.create(Node.SliceType);
- node.* = .{
- .op_token = lbracket,
- .rhs = undefined, // set by caller
- .ptr_info = .{ .sentinel = sentinel },
- };
- return &node.base;
+ return expr;
}
- /// PtrTypeStart
- /// <- ASTERISK
- /// / ASTERISK2
- /// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET
- fn parsePtrTypeStart(p: *Parser) !?*Node {
- if (p.eatToken(.Asterisk)) |asterisk| {
- const sentinel = if (p.eatToken(.Colon)) |_|
- try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- })
- else
- null;
- const node = try p.arena.allocator.create(Node.PtrType);
- node.* = .{
- .op_token = asterisk,
- .rhs = undefined, // set by caller
- .ptr_info = .{ .sentinel = sentinel },
- };
- return &node.base;
- }
+ const PtrModifiers = struct {
+ align_node: Node.Index,
+ bit_range_start: Node.Index,
+ bit_range_end: Node.Index,
+ };
- if (p.eatToken(.AsteriskAsterisk)) |double_asterisk| {
- const node = try p.arena.allocator.create(Node.PtrType);
- node.* = .{
- .op_token = double_asterisk,
- .rhs = undefined, // set by caller
- };
+ fn parsePtrModifiers(p: *Parser) !PtrModifiers {
+ var result: PtrModifiers = .{
+ .align_node = 0,
+ .bit_range_start = 0,
+ .bit_range_end = 0,
+ };
+ var saw_const = false;
+ var saw_volatile = false;
+ var saw_allowzero = false;
+ while (true) {
+ switch (p.token_tags[p.tok_i]) {
+ .Keyword_align => {
+ if (result.align_node != 0) {
+ try p.warn(.{
+ .ExtraAlignQualifier = .{ .token = p.tok_i },
+ });
+ }
+ p.tok_i += 1;
+ _ = try p.expectToken(.LParen);
+ result.align_node = try p.expectExpr();
- // Special case for **, which is its own token
- const child = try p.arena.allocator.create(Node.PtrType);
- child.* = .{
- .op_token = double_asterisk,
- .rhs = undefined, // set by caller
- };
- node.rhs = &child.base;
+ if (p.eatToken(.Colon)) |_| {
+ result.bit_range_start = try p.expectExpr();
+ _ = try p.expectToken(.Colon);
+ result.bit_range_end = try p.expectExpr();
+ }
- return &node.base;
- }
- if (p.eatToken(.LBracket)) |lbracket| {
- const asterisk = p.eatToken(.Asterisk) orelse {
- p.putBackToken(lbracket);
- return null;
- };
- if (p.eatToken(.Identifier)) |ident| {
- const token_loc = p.token_locs[ident];
- const token_slice = p.source[token_loc.start..token_loc.end];
- if (!std.mem.eql(u8, token_slice, "c")) {
- p.putBackToken(ident);
- } else {
- _ = try p.expectToken(.RBracket);
- const node = try p.arena.allocator.create(Node.PtrType);
- node.* = .{
- .op_token = lbracket,
- .rhs = undefined, // set by caller
- };
- return &node.base;
- }
+ _ = try p.expectToken(.RParen);
+ },
+ .Keyword_const => {
+ if (saw_const) {
+ try p.warn(.{
+ .ExtraConstQualifier = .{ .token = p.tok_i },
+ });
+ }
+ p.tok_i += 1;
+ saw_const = true;
+ },
+ .Keyword_volatile => {
+ if (saw_volatile) {
+ try p.warn(.{
+ .ExtraVolatileQualifier = .{ .token = p.tok_i },
+ });
+ }
+ p.tok_i += 1;
+ saw_volatile = true;
+ },
+ .Keyword_allowzero => {
+ if (saw_allowzero) {
+ try p.warn(.{
+ .ExtraAllowZeroQualifier = .{ .token = p.tok_i },
+ });
+ }
+ p.tok_i += 1;
+ saw_allowzero = true;
+ },
+ else => return result,
}
- const sentinel = if (p.eatToken(.Colon)) |_|
- try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- })
- else
- null;
- _ = try p.expectToken(.RBracket);
- const node = try p.arena.allocator.create(Node.PtrType);
- node.* = .{
- .op_token = lbracket,
- .rhs = undefined, // set by caller
- .ptr_info = .{ .sentinel = sentinel },
- };
- return &node.base;
}
- return null;
}
- /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
- fn parseContainerDeclAuto(p: *Parser) !?*Node {
- const container_decl_type = (try p.parseContainerDeclType()) orelse return null;
- const lbrace = try p.expectToken(.LBrace);
- const members = try p.parseContainerMembers(false);
- defer p.gpa.free(members);
- const rbrace = try p.expectToken(.RBrace);
-
- const members_len = @intCast(NodeIndex, members.len);
- const node = try Node.ContainerDecl.alloc(&p.arena.allocator, members_len);
- node.* = .{
- .layout_token = null,
- .kind_token = container_decl_type.kind_token,
- .init_arg_expr = container_decl_type.init_arg_expr,
- .fields_and_decls_len = members_len,
- .lbrace_token = lbrace,
- .rbrace_token = rbrace,
- };
- std.mem.copy(*Node, node.fieldsAndDecls(), members);
- return &node.base;
+ /// SuffixOp
+ /// <- LBRACKET Expr (DOT2 (Expr (COLON Expr)?)?)? RBRACKET
+ /// / DOT IDENTIFIER
+ /// / DOTASTERISK
+ /// / DOTQUESTIONMARK
+ fn parseSuffixOp(p: *Parser, lhs: Node.Index) !Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .LBracket => {
+ const lbracket = p.nextToken();
+ const index_expr = try p.expectExpr();
+
+ if (p.eatToken(.Ellipsis2)) |_| {
+ const end_expr = try p.parseExpr();
+ if (end_expr == 0) {
+ _ = try p.expectToken(.RBracket);
+ return p.addNode(.{
+ .tag = .SliceOpen,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = lhs,
+ .rhs = index_expr,
+ },
+ });
+ }
+ const sentinel: Node.Index = if (p.eatToken(.Colon)) |_|
+ try p.parseExpr()
+ else
+ 0;
+ _ = try p.expectToken(.RBracket);
+ return p.addNode(.{
+ .tag = .Slice,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = lhs,
+ .rhs = try p.addExtra(.{
+ .start = index_expr,
+ .end = end_expr,
+ .sentinel = sentinel,
+ }),
+ },
+ });
+ }
+ _ = try p.expectToken(.RBracket);
+ return p.addNode(.{
+ .tag = .ArrayAccess,
+ .main_token = lbracket,
+ .data = .{
+ .lhs = lhs,
+ .rhs = index_expr,
+ },
+ });
+ },
+ .PeriodAsterisk => return p.addNode(.{
+ .tag = .Deref,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = lhs,
+ .rhs = undefined,
+ },
+ }),
+ .Invalid_periodasterisks => {
+ const period_asterisk = p.nextToken();
+ try p.warn(.{ .AsteriskAfterPointerDereference = .{ .token = period_asterisk } });
+ return p.addNode(.{
+ .tag = .Deref,
+ .main_token = period_asterisk,
+ .data = .{
+ .lhs = lhs,
+ .rhs = undefined,
+ },
+ });
+ },
+ .Period => switch (p.token_tags[p.tok_i + 1]) {
+ .Identifier => return p.addNode(.{
+ .tag = .FieldAccess,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = lhs,
+ .rhs = p.nextToken(),
+ },
+ }),
+ .QuestionMark => return p.addNode(.{
+ .tag = .UnwrapOptional,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = lhs,
+ .rhs = p.nextToken(),
+ },
+ }),
+ else => {
+ p.tok_i += 1;
+ try p.warn(.{ .ExpectedSuffixOp = .{ .token = p.tok_i } });
+ return null_node;
+ },
+ },
+ else => return null_node,
+ }
}
- /// Holds temporary data until we are ready to construct the full ContainerDecl AST node.
- const ContainerDeclType = struct {
- kind_token: TokenIndex,
- init_arg_expr: Node.ContainerDecl.InitArg,
- };
-
+ /// Caller must have already verified the first token.
/// ContainerDeclType
/// <- KEYWORD_struct
/// / KEYWORD_enum (LPAREN Expr RPAREN)?
/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)?
/// / KEYWORD_opaque
- fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
- const kind_token = p.nextToken();
-
- const init_arg_expr = switch (p.token_ids[kind_token]) {
- .Keyword_struct, .Keyword_opaque => Node.ContainerDecl.InitArg{ .None = {} },
+ fn parseContainerDeclAuto(p: *Parser) !Node.Index {
+ const main_token = p.nextToken();
+ const arg_expr = switch (p.token_tags[main_token]) {
+ .Keyword_struct, .Keyword_opaque => null_node,
.Keyword_enum => blk: {
- if (p.eatToken(.LParen) != null) {
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ if (p.eatToken(.LParen)) |_| {
+ const expr = try p.expectExpr();
_ = try p.expectToken(.RParen);
- break :blk Node.ContainerDecl.InitArg{ .Type = expr };
+ break :blk expr;
+ } else {
+ break :blk null_node;
}
- break :blk Node.ContainerDecl.InitArg{ .None = {} };
},
.Keyword_union => blk: {
- if (p.eatToken(.LParen) != null) {
- if (p.eatToken(.Keyword_enum) != null) {
- if (p.eatToken(.LParen) != null) {
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ if (p.eatToken(.LParen)) |_| {
+ if (p.eatToken(.Keyword_enum)) |_| {
+ if (p.eatToken(.LParen)) |_| {
+ const enum_tag_expr = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
_ = try p.expectToken(.RParen);
+
+ _ = try p.expectToken(.LBrace);
+ const members = try p.parseContainerMembers(false);
+ _ = try p.expectToken(.RBrace);
+ return p.addNode(.{
+ .tag = .TaggedUnionEnumTag,
+ .main_token = main_token,
+ .data = .{
+ .lhs = enum_tag_expr,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = members.start,
+ .end = members.end,
+ }),
+ },
+ });
+ } else {
_ = try p.expectToken(.RParen);
- break :blk Node.ContainerDecl.InitArg{ .Enum = expr };
+
+ _ = try p.expectToken(.LBrace);
+ const members = try p.parseContainerMembers(false);
+ _ = try p.expectToken(.RBrace);
+ return p.addNode(.{
+ .tag = .TaggedUnion,
+ .main_token = main_token,
+ .data = .{
+ .lhs = members.start,
+ .rhs = members.end,
+ },
+ });
}
+ } else {
+ const expr = try p.expectExpr();
_ = try p.expectToken(.RParen);
- break :blk Node.ContainerDecl.InitArg{ .Enum = null };
+ break :blk expr;
}
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
- _ = try p.expectToken(.RParen);
- break :blk Node.ContainerDecl.InitArg{ .Type = expr };
+ } else {
+ break :blk null_node;
}
- break :blk Node.ContainerDecl.InitArg{ .None = {} };
- },
- else => {
- p.putBackToken(kind_token);
- return null;
},
+ else => unreachable,
};
-
- return ContainerDeclType{
- .kind_token = kind_token,
- .init_arg_expr = init_arg_expr,
- };
+ _ = try p.expectToken(.LBrace);
+ const members = try p.parseContainerMembers(false);
+ _ = try p.expectToken(.RBrace);
+ if (arg_expr == 0) {
+ return p.addNode(.{
+ .tag = .ContainerDecl,
+ .main_token = main_token,
+ .data = .{
+ .lhs = members.start,
+ .rhs = members.end,
+ },
+ });
+ } else {
+ return p.addNode(.{
+ .tag = .ContainerDeclArg,
+ .main_token = main_token,
+ .data = .{
+ .lhs = arg_expr,
+ .rhs = try p.addExtra(Node.SubRange{
+ .start = members.start,
+ .end = members.end,
+ }),
+ },
+ });
+ }
}
+ /// Holds temporary data until we are ready to construct the full ContainerDecl AST node.
/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN
- fn parseByteAlign(p: *Parser) !?*Node {
- _ = p.eatToken(.Keyword_align) orelse return null;
+ fn parseByteAlign(p: *Parser) !Node.Index {
+ _ = p.eatToken(.Keyword_align) orelse return null_node;
_ = try p.expectToken(.LParen);
- const expr = try p.expectNode(parseExpr, .{
- .ExpectedExpr = .{ .token = p.tok_i },
- });
+ const expr = try p.expectExpr();
_ = try p.expectToken(.RParen);
return expr;
}
- /// IdentifierList <- (IDENTIFIER COMMA)* IDENTIFIER?
- /// Only ErrorSetDecl parses an IdentifierList
- fn parseErrorTagList(p: *Parser) ![]*Node {
- return ListParseFn(*Node, parseErrorTag)(p);
- }
-
/// SwitchProngList <- (SwitchProng COMMA)* SwitchProng?
- fn parseSwitchProngList(p: *Parser) ![]*Node {
- return ListParseFn(*Node, parseSwitchProng)(p);
+ fn parseSwitchProngList(p: *Parser) !Node.SubRange {
+ return ListParseFn(parseSwitchProng)(p);
}
- /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem?
- fn parseAsmOutputList(p: *Parser) Error![]Node.Asm.Output {
- return ListParseFn(Node.Asm.Output, parseAsmOutputItem)(p);
- }
+ /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl?
+ fn parseParamDeclList(p: *Parser) !SmallSpan {
+ _ = try p.expectToken(.LParen);
+ if (p.eatToken(.RParen)) |_| {
+ return SmallSpan{ .zero_or_one = 0 };
+ }
+ const param_one = while (true) {
+ const param = try p.expectParamDecl();
+ if (param != 0) break param;
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => continue,
+ .RParen => return SmallSpan{ .zero_or_one = 0 },
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ } else unreachable;
- /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem?
- fn parseAsmInputList(p: *Parser) Error![]Node.Asm.Input {
- return ListParseFn(Node.Asm.Input, parseAsmInputItem)(p);
- }
+ const param_two = while (true) {
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => {
+ if (p.eatToken(.RParen)) |_| {
+ return SmallSpan{ .zero_or_one = param_one };
+ }
+ const param = try p.expectParamDecl();
+ if (param != 0) break param;
+ continue;
+ },
+ .RParen => return SmallSpan{ .zero_or_one = param_one },
+ .Colon, .RBrace, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .RParen },
+ });
+ },
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ } else unreachable;
- /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl?
- fn parseParamDeclList(p: *Parser) ![]Node.FnProto.ParamDecl {
- return ListParseFn(Node.FnProto.ParamDecl, parseParamDecl)(p);
+ var list = std.ArrayList(Node.Index).init(p.gpa);
+ defer list.deinit();
+
+ try list.appendSlice(&[_]Node.Index{ param_one, param_two });
+
+ while (true) {
+ switch (p.token_tags[p.nextToken()]) {
+ .Comma => {
+ if (p.token_tags[p.tok_i] == .RParen) {
+ p.tok_i += 1;
+ return SmallSpan{ .multi = list.toOwnedSlice() };
+ }
+ const param = try p.expectParamDecl();
+ if (param != 0) {
+ try list.append(param);
+ }
+ continue;
+ },
+ .RParen => return SmallSpan{ .multi = list.toOwnedSlice() },
+ .Colon, .RBrace, .RBracket => {
+ p.tok_i -= 1;
+ return p.fail(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .RParen },
+ });
+ },
+ else => {
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ p.tok_i -= 1;
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
+ });
+ },
+ }
+ }
}
- const NodeParseFn = fn (p: *Parser) Error!?*Node;
+ const NodeParseFn = fn (p: *Parser) Error!Node.Index;
- fn ListParseFn(comptime E: type, comptime nodeParseFn: anytype) ParseFn([]E) {
+ fn ListParseFn(comptime nodeParseFn: anytype) (fn (p: *Parser) Error!Node.SubRange) {
return struct {
- pub fn parse(p: *Parser) ![]E {
- var list = std.ArrayList(E).init(p.gpa);
+ pub fn parse(p: *Parser) Error!Node.SubRange {
+ var list = std.ArrayList(Node.Index).init(p.gpa);
defer list.deinit();
- while (try nodeParseFn(p)) |item| {
+ while (true) {
+ const item = try nodeParseFn(p);
+ if (item == 0) break;
+
try list.append(item);
- switch (p.token_ids[p.tok_i]) {
- .Comma => _ = p.nextToken(),
+ switch (p.token_tags[p.tok_i]) {
+ .Comma => p.tok_i += 1,
// all possible delimiters
.Colon, .RParen, .RBrace, .RBracket => break,
else => {
- // this is likely just a missing comma,
- // continue parsing this list and give an error
- try p.errors.append(p.gpa, .{
+ // This is likely just a missing comma;
+ // give an error but continue parsing this list.
+ try p.warn(.{
.ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma },
});
},
}
}
- return list.toOwnedSlice();
- }
- }.parse;
- }
-
- fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.Tag) NodeParseFn {
- return struct {
- pub fn parse(p: *Parser) Error!?*Node {
- const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) {
- .Keyword_and => p.nextToken(),
- .Invalid_ampersands => blk: {
- try p.errors.append(p.gpa, .{
- .InvalidAnd = .{ .token = p.tok_i },
- });
- break :blk p.nextToken();
- },
- else => return null,
- } else p.eatToken(token) orelse return null;
-
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = .{ .tag = op },
- .op_token = op_token,
- .lhs = undefined, // set by caller
- .rhs = undefined, // set by caller
- };
- return &node.base;
+ return p.listToSpan(list.items);
}
}.parse;
}
- // Helper parsers not included in the grammar
+ /// FnCallArguments <- LPAREN ExprList RPAREN
+ /// ExprList <- (Expr COMMA)* Expr?
+ /// TODO detect when we can emit BuiltinCallTwo instead of BuiltinCall.
+ fn parseBuiltinCall(p: *Parser) !Node.Index {
+ const builtin_token = p.eatToken(.Builtin) orelse return null_node;
- fn parseBuiltinCall(p: *Parser) !?*Node {
- const token = p.eatToken(.Builtin) orelse return null;
- const params = (try p.parseFnCallArguments()) orelse {
- try p.errors.append(p.gpa, .{
+ const lparen = (try p.expectTokenRecoverable(.LParen)) orelse {
+ try p.warn(.{
.ExpectedParamList = .{ .token = p.tok_i },
});
-
- // lets pretend this was an identifier so we can continue parsing
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .Identifier },
- .token = token,
- };
- return &node.base;
- };
- defer p.gpa.free(params.list);
-
- const node = try Node.BuiltinCall.alloc(&p.arena.allocator, params.list.len);
- node.* = .{
- .builtin_token = token,
- .params_len = params.list.len,
- .rparen_token = params.rparen,
- };
- std.mem.copy(*Node, node.params(), params.list);
- return &node.base;
- }
-
- fn parseErrorTag(p: *Parser) !?*Node {
- const doc_comments = try p.parseDocComment(); // no need to rewind on failure
- const token = p.eatToken(.Identifier) orelse return null;
-
- const node = try p.arena.allocator.create(Node.ErrorTag);
- node.* = .{
- .doc_comments = doc_comments,
- .name_token = token,
- };
- return &node.base;
- }
-
- fn parseIdentifier(p: *Parser) !?*Node {
- const token = p.eatToken(.Identifier) orelse return null;
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .Identifier },
- .token = token,
+ // Pretend this was an identifier so we can continue parsing.
+ return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = builtin_token,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
};
- return &node.base;
+ const params = try ListParseFn(parseExpr)(p);
+ _ = try p.expectToken(.RParen);
+ return p.addNode(.{
+ .tag = .BuiltinCall,
+ .main_token = builtin_token,
+ .data = .{
+ .lhs = params.start,
+ .rhs = params.end,
+ },
+ });
}
- fn parseAnyType(p: *Parser) !?*Node {
- const token = p.eatToken(.Keyword_anytype) orelse
- p.eatToken(.Keyword_var) orelse return null; // TODO remove in next release cycle
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .AnyType },
- .token = token,
- };
- return &node.base;
+ fn parseOneToken(p: *Parser, token_tag: Token.Tag) !Node.Index {
+ const token = p.eatToken(token_tag) orelse return null_node;
+ return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = token,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ });
}
- fn createLiteral(p: *Parser, tag: ast.Node.Tag, token: TokenIndex) !*Node {
- const result = try p.arena.allocator.create(Node.OneToken);
- result.* = .{
- .base = .{ .tag = tag },
- .token = token,
- };
- return &result.base;
+ fn expectOneToken(p: *Parser, token_tag: Token.Tag) !Node.Index {
+ const node = try p.expectOneTokenRecoverable(token_tag);
+ if (node == 0) return error.ParseError;
+ return node;
}
- fn parseStringLiteralSingle(p: *Parser) !?*Node {
- if (p.eatToken(.StringLiteral)) |token| {
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .StringLiteral },
- .token = token,
- };
- return &node.base;
+ fn expectOneTokenRecoverable(p: *Parser, token_tag: Token.Tag) !Node.Index {
+ const node = p.parseOneToken(token_tag);
+ if (node == 0) {
+ try p.warn(.{
+ .ExpectedToken = .{
+ .token = p.tok_i,
+ .expected_id = token_tag,
+ },
+ });
}
- return null;
+ return node;
}
// string literal or multiline string literal
- fn parseStringLiteral(p: *Parser) !?*Node {
- if (try p.parseStringLiteralSingle()) |node| return node;
-
- if (p.eatToken(.MultilineStringLiteralLine)) |first_line| {
- const start_tok_i = p.tok_i;
- var tok_i = start_tok_i;
- var count: usize = 1; // including first_line
- while (true) : (tok_i += 1) {
- switch (p.token_ids[tok_i]) {
- .LineComment => continue,
- .MultilineStringLiteralLine => count += 1,
- else => break,
+ fn parseStringLiteral(p: *Parser) !Node.Index {
+ switch (p.token_tags[p.tok_i]) {
+ .StringLiteral => return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = p.nextToken(),
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
+ },
+ }),
+ .MultilineStringLiteralLine => {
+ const first_line = p.nextToken();
+ while (p.token_tags[p.tok_i] == .MultilineStringLiteralLine) {
+ p.tok_i += 1;
}
- }
-
- const node = try Node.MultilineStringLiteral.alloc(&p.arena.allocator, count);
- node.* = .{ .lines_len = count };
- const lines = node.lines();
- tok_i = start_tok_i;
- lines[0] = first_line;
- count = 1;
- while (true) : (tok_i += 1) {
- switch (p.token_ids[tok_i]) {
- .LineComment => continue,
- .MultilineStringLiteralLine => {
- lines[count] = tok_i;
- count += 1;
+ return p.addNode(.{
+ .tag = .OneToken,
+ .main_token = first_line,
+ .data = .{
+ .lhs = undefined,
+ .rhs = undefined,
},
- else => break,
- }
- }
- p.tok_i = tok_i;
- return &node.base;
+ });
+ },
+ else => return null_node,
}
-
- return null;
}
- fn parseIntegerLiteral(p: *Parser) !?*Node {
- const token = p.eatToken(.IntegerLiteral) orelse return null;
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .IntegerLiteral },
- .token = token,
- };
- return &node.base;
+ fn expectStringLiteral(p: *Parser) !Node.Index {
+ const node = try p.parseStringLiteral();
+ if (node == 0) {
+ return p.fail(.{ .ExpectedStringLiteral = .{ .token = p.tok_i } });
+ }
+ return node;
}
- fn parseFloatLiteral(p: *Parser) !?*Node {
- const token = p.eatToken(.FloatLiteral) orelse return null;
- const node = try p.arena.allocator.create(Node.OneToken);
- node.* = .{
- .base = .{ .tag = .FloatLiteral },
- .token = token,
- };
- return &node.base;
+ fn expectIntegerLiteral(p: *Parser) !Node.Index {
+ const node = p.parseOneToken(.IntegerLiteral);
+ if (node != 0) {
+ return p.fail(.{ .ExpectedIntegerLiteral = .{ .token = p.tok_i } });
+ }
+ return node;
}
- fn parseTry(p: *Parser) !?*Node {
- const token = p.eatToken(.Keyword_try) orelse return null;
- const node = try p.arena.allocator.create(Node.SimplePrefixOp);
- node.* = .{
- .base = .{ .tag = .Try },
- .op_token = token,
- .rhs = undefined, // set by caller
- };
- return &node.base;
- }
+ /// KEYWORD_if LPAREN Expr RPAREN PtrPayload? Body (KEYWORD_else Payload? Body)?
+ fn parseIf(p: *Parser, bodyParseFn: NodeParseFn) !Node.Index {
+ const if_token = p.eatToken(.Keyword_if) orelse return null_node;
+ _ = try p.expectToken(.LParen);
+ const condition = try p.expectExpr();
+ _ = try p.expectToken(.RParen);
+ const then_payload = try p.parsePtrPayload();
- /// IfPrefix Body (KEYWORD_else Payload? Body)?
- fn parseIf(p: *Parser, bodyParseFn: NodeParseFn) !?*Node {
- const node = (try p.parseIfPrefix()) orelse return null;
- const if_prefix = node.cast(Node.If).?;
+ const then_expr = try bodyParseFn(p);
+ if (then_expr == 0) return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
- if_prefix.body = try p.expectNode(bodyParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
+ const else_token = p.eatToken(.Keyword_else) orelse return p.addNode(.{
+ .tag = if (then_payload == 0) .IfSimple else .IfSimpleOptional,
+ .main_token = if_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = then_expr,
+ },
});
-
- const else_token = p.eatToken(.Keyword_else) orelse return node;
- const payload = try p.parsePayload();
- const else_expr = try p.expectNode(bodyParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
+ const else_payload = try p.parsePayload();
+ const else_expr = try bodyParseFn(p);
+ if (else_expr == 0) return p.fail(.{ .InvalidToken = .{ .token = p.tok_i } });
+
+ const tag = if (else_payload != 0)
+ Node.Tag.IfError
+ else if (then_payload != 0)
+ Node.Tag.IfOptional
+ else
+ Node.Tag.If;
+ return p.addNode(.{
+ .tag = tag,
+ .main_token = if_token,
+ .data = .{
+ .lhs = condition,
+ .rhs = try p.addExtra(Node.If{
+ .then_expr = then_expr,
+ .else_expr = else_expr,
+ }),
+ },
});
- const else_node = try p.arena.allocator.create(Node.Else);
- else_node.* = .{
- .else_token = else_token,
- .payload = payload,
- .body = else_expr,
- };
- if_prefix.@"else" = else_node;
-
- return node;
}
- /// Eat a multiline doc comment
- fn parseDocComment(p: *Parser) !?*Node.DocComment {
+ /// Skips over doc comment tokens. Returns the first one, if any.
+ fn eatDocComments(p: *Parser) ?TokenIndex {
if (p.eatToken(.DocComment)) |first_line| {
while (p.eatToken(.DocComment)) |_| {}
- const node = try p.arena.allocator.create(Node.DocComment);
- node.* = .{ .first_line = first_line };
- return node;
+ return first_line;
}
return null;
}
fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
- return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null;
+ return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null;
}
/// Eat a single-line doc comment on the same line as another node
- fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment {
- const comment_token = p.eatToken(.DocComment) orelse return null;
- if (p.tokensOnSameLine(after_token, comment_token)) {
- const node = try p.arena.allocator.create(Node.DocComment);
- node.* = .{ .first_line = comment_token };
- return node;
- }
- p.putBackToken(comment_token);
- return null;
- }
-
- /// Op* Child
- fn parsePrefixOpExpr(p: *Parser, comptime opParseFn: NodeParseFn, comptime childParseFn: NodeParseFn) Error!?*Node {
- if (try opParseFn(p)) |first_op| {
- var rightmost_op = first_op;
- while (true) {
- switch (rightmost_op.tag) {
- .AddressOf,
- .Await,
- .BitNot,
- .BoolNot,
- .OptionalType,
- .Negation,
- .NegationWrap,
- .Resume,
- .Try,
- => {
- if (try opParseFn(p)) |rhs| {
- rightmost_op.cast(Node.SimplePrefixOp).?.rhs = rhs;
- rightmost_op = rhs;
- } else break;
- },
- .ArrayType => {
- if (try opParseFn(p)) |rhs| {
- rightmost_op.cast(Node.ArrayType).?.rhs = rhs;
- rightmost_op = rhs;
- } else break;
- },
- .ArrayTypeSentinel => {
- if (try opParseFn(p)) |rhs| {
- rightmost_op.cast(Node.ArrayTypeSentinel).?.rhs = rhs;
- rightmost_op = rhs;
- } else break;
- },
- .SliceType => {
- if (try opParseFn(p)) |rhs| {
- rightmost_op.cast(Node.SliceType).?.rhs = rhs;
- rightmost_op = rhs;
- } else break;
- },
- .PtrType => {
- var ptr_type = rightmost_op.cast(Node.PtrType).?;
- // If the token encountered was **, there will be two nodes
- if (p.token_ids[ptr_type.op_token] == .AsteriskAsterisk) {
- rightmost_op = ptr_type.rhs;
- ptr_type = rightmost_op.cast(Node.PtrType).?;
- }
- if (try opParseFn(p)) |rhs| {
- ptr_type.rhs = rhs;
- rightmost_op = rhs;
- } else break;
- },
- .AnyFrameType => {
- const prom = rightmost_op.cast(Node.AnyFrameType).?;
- if (try opParseFn(p)) |rhs| {
- prom.result.?.return_type = rhs;
- rightmost_op = rhs;
- } else break;
- },
- else => unreachable,
- }
- }
-
- // If any prefix op existed, a child node on the RHS is required
- switch (rightmost_op.tag) {
- .AddressOf,
- .Await,
- .BitNot,
- .BoolNot,
- .OptionalType,
- .Negation,
- .NegationWrap,
- .Resume,
- .Try,
- => {
- const prefix_op = rightmost_op.cast(Node.SimplePrefixOp).?;
- prefix_op.rhs = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- .ArrayType => {
- const prefix_op = rightmost_op.cast(Node.ArrayType).?;
- prefix_op.rhs = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- .ArrayTypeSentinel => {
- const prefix_op = rightmost_op.cast(Node.ArrayTypeSentinel).?;
- prefix_op.rhs = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- .PtrType => {
- const prefix_op = rightmost_op.cast(Node.PtrType).?;
- prefix_op.rhs = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- .SliceType => {
- const prefix_op = rightmost_op.cast(Node.SliceType).?;
- prefix_op.rhs = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- .AnyFrameType => {
- const prom = rightmost_op.cast(Node.AnyFrameType).?;
- prom.result.?.return_type = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- },
- else => unreachable,
- }
-
- return first_op;
- }
-
- // Otherwise, the child node is optional
- return childParseFn(p);
- }
-
- /// Child (Op Child)*
- /// Child (Op Child)?
- fn parseBinOpExpr(
- p: *Parser,
- opParseFn: NodeParseFn,
- childParseFn: NodeParseFn,
- chain: enum {
- Once,
- Infinitely,
- },
- ) Error!?*Node {
- var res = (try childParseFn(p)) orelse return null;
-
- while (try opParseFn(p)) |node| {
- const right = try p.expectNode(childParseFn, .{
- .InvalidToken = .{ .token = p.tok_i },
- });
- const left = res;
- res = node;
-
- if (node.castTag(.Catch)) |op| {
- op.lhs = left;
- op.rhs = right;
- } else if (node.cast(Node.SimpleInfixOp)) |op| {
- op.lhs = left;
- op.rhs = right;
- }
-
- switch (chain) {
- .Once => break,
- .Infinitely => continue,
- }
+ fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !void {
+ const comment_token = p.eatToken(.DocComment) orelse return;
+ if (!p.tokensOnSameLine(after_token, comment_token)) {
+ p.tok_i -= 1;
}
-
- return res;
}
- fn createInfixOp(p: *Parser, op_token: TokenIndex, tag: Node.Tag) !*Node {
- const node = try p.arena.allocator.create(Node.SimpleInfixOp);
- node.* = .{
- .base = Node{ .tag = tag },
- .op_token = op_token,
- .lhs = undefined, // set by caller
- .rhs = undefined, // set by caller
- };
- return &node.base;
+ fn eatToken(p: *Parser, tag: Token.Tag) ?TokenIndex {
+ return if (p.token_tags[p.tok_i] == tag) p.nextToken() else null;
}
- fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {
- return if (p.token_ids[p.tok_i] == id) p.nextToken() else null;
+ fn assertToken(p: *Parser, tag: Token.Tag) TokenIndex {
+ const token = p.nextToken();
+ assert(p.token_tags[token] == tag);
+ return token;
}
- fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {
- return (try p.expectTokenRecoverable(id)) orelse error.ParseError;
+ fn expectToken(p: *Parser, tag: Token.Tag) Error!TokenIndex {
+ const token = p.nextToken();
+ if (p.token_tags[token] != tag) {
+ return p.fail(.{ .ExpectedToken = .{ .token = token, .expected_id = tag } });
+ }
+ return token;
}
- fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {
- const token = p.nextToken();
- if (p.token_ids[token] != id) {
- try p.errors.append(p.gpa, .{
- .ExpectedToken = .{ .token = token, .expected_id = id },
+ fn expectTokenRecoverable(p: *Parser, tag: Token.Tag) !?TokenIndex {
+ if (p.token_tags[p.tok_i] != tag) {
+ try p.warn(.{
+ .ExpectedToken = .{ .token = p.tok_i, .expected_id = tag },
});
- // go back so that we can recover properly
- p.putBackToken(token);
return null;
+ } else {
+ return p.nextToken();
}
- return token;
}
fn nextToken(p: *Parser) TokenIndex {
const result = p.tok_i;
p.tok_i += 1;
- assert(p.token_ids[result] != .LineComment);
- if (p.tok_i >= p.token_ids.len) return result;
-
- while (true) {
- if (p.token_ids[p.tok_i] != .LineComment) return result;
- p.tok_i += 1;
- }
- }
-
- fn putBackToken(p: *Parser, putting_back: TokenIndex) void {
- while (p.tok_i > 0) {
- p.tok_i -= 1;
- if (p.token_ids[p.tok_i] == .LineComment) continue;
- assert(putting_back == p.tok_i);
- return;
- }
- }
-
- /// TODO Delete this function. I don't like the inversion of control.
- fn expectNode(
- p: *Parser,
- parseFn: NodeParseFn,
- /// if parsing fails
- err: AstError,
- ) Error!*Node {
- return (try p.expectNodeRecoverable(parseFn, err)) orelse return error.ParseError;
- }
-
- /// TODO Delete this function. I don't like the inversion of control.
- fn expectNodeRecoverable(
- p: *Parser,
- parseFn: NodeParseFn,
- /// if parsing fails
- err: AstError,
- ) !?*Node {
- return (try parseFn(p)) orelse {
- try p.errors.append(p.gpa, err);
- return null;
- };
+ return result;
}
};
-fn ParseFn(comptime T: type) type {
- return fn (p: *Parser) Error!T;
-}
-
-test "std.zig.parser" {
+test {
_ = @import("parser_test.zig");
}
lib/std/zig/parser_test.zig
@@ -3736,12 +3736,13 @@ var fixed_buffer_mem: [100 * 1024]u8 = undefined;
fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 {
const stderr = io.getStdErr().writer();
- const tree = try std.zig.parse(allocator, source);
- defer tree.deinit();
+ var tree = try std.zig.parse(allocator, source);
+ defer tree.deinit(allocator);
- for (tree.errors) |*parse_error| {
- const token = tree.token_locs[parse_error.loc()];
- const loc = tree.tokenLocation(0, parse_error.loc());
+ for (tree.errors) |parse_error| {
+ const error_token = tree.errorToken(parse_error);
+ const token_start = tree.tokens.items(.start)[error_token];
+ const loc = tree.tokenLocation(0, error_token);
try stderr.print("(memory buffer):{d}:{d}: error: ", .{ loc.line + 1, loc.column + 1 });
try tree.renderError(parse_error, stderr);
try stderr.print("\n{s}\n", .{source[loc.line_start..loc.line_end]});
@@ -3750,13 +3751,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
while (i < loc.column) : (i += 1) {
try stderr.writeAll(" ");
}
- }
- {
- const caret_count = token.end - token.start;
- var i: usize = 0;
- while (i < caret_count) : (i += 1) {
- try stderr.writeAll("~");
- }
+ try stderr.writeAll("^");
}
try stderr.writeAll("\n");
}
@@ -3825,8 +3820,8 @@ fn testCanonical(source: []const u8) !void {
const Error = @TagType(std.zig.ast.Error);
fn testError(source: []const u8, expected_errors: []const Error) !void {
- const tree = try std.zig.parse(std.testing.allocator, source);
- defer tree.deinit();
+ var tree = try std.zig.parse(std.testing.allocator, source);
+ defer tree.deinit(std.testing.allocator);
std.testing.expect(tree.errors.len == expected_errors.len);
for (expected_errors) |expected, i| {
lib/std/zig/tokenizer.zig
@@ -7,7 +7,7 @@ const std = @import("../std.zig");
const mem = std.mem;
pub const Token = struct {
- id: Id,
+ tag: Tag,
loc: Loc,
pub const Loc = struct {
@@ -15,7 +15,7 @@ pub const Token = struct {
end: usize,
};
- pub const keywords = std.ComptimeStringMap(Id, .{
+ pub const keywords = std.ComptimeStringMap(Tag, .{
.{ "align", .Keyword_align },
.{ "allowzero", .Keyword_allowzero },
.{ "and", .Keyword_and },
@@ -71,11 +71,11 @@ pub const Token = struct {
.{ "while", .Keyword_while },
});
- pub fn getKeyword(bytes: []const u8) ?Id {
+ pub fn getKeyword(bytes: []const u8) ?Tag {
return keywords.get(bytes);
}
- pub const Id = enum {
+ pub const Tag = enum {
Invalid,
Invalid_ampersands,
Invalid_periodasterisks,
@@ -198,8 +198,8 @@ pub const Token = struct {
Keyword_volatile,
Keyword_while,
- pub fn symbol(id: Id) []const u8 {
- return switch (id) {
+ pub fn symbol(tag: Tag) []const u8 {
+ return switch (tag) {
.Invalid => "Invalid",
.Invalid_ampersands => "&&",
.Invalid_periodasterisks => ".**",
@@ -334,7 +334,7 @@ pub const Tokenizer = struct {
/// For debugging purposes
pub fn dump(self: *Tokenizer, token: *const Token) void {
- std.debug.warn("{s} \"{s}\"\n", .{ @tagName(token.id), self.buffer[token.start..token.end] });
+ std.debug.warn("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.start..token.end] });
}
pub fn init(buffer: []const u8) Tokenizer {
@@ -421,7 +421,7 @@ pub const Tokenizer = struct {
const start_index = self.index;
var state: State = .start;
var result = Token{
- .id = .Eof,
+ .tag = .Eof,
.loc = .{
.start = self.index,
.end = undefined,
@@ -438,14 +438,14 @@ pub const Tokenizer = struct {
},
'"' => {
state = .string_literal;
- result.id = .StringLiteral;
+ result.tag = .StringLiteral;
},
'\'' => {
state = .char_literal;
},
'a'...'z', 'A'...'Z', '_' => {
state = .identifier;
- result.id = .Identifier;
+ result.tag = .Identifier;
},
'@' => {
state = .saw_at_sign;
@@ -460,42 +460,42 @@ pub const Tokenizer = struct {
state = .pipe;
},
'(' => {
- result.id = .LParen;
+ result.tag = .LParen;
self.index += 1;
break;
},
')' => {
- result.id = .RParen;
+ result.tag = .RParen;
self.index += 1;
break;
},
'[' => {
- result.id = .LBracket;
+ result.tag = .LBracket;
self.index += 1;
break;
},
']' => {
- result.id = .RBracket;
+ result.tag = .RBracket;
self.index += 1;
break;
},
';' => {
- result.id = .Semicolon;
+ result.tag = .Semicolon;
self.index += 1;
break;
},
',' => {
- result.id = .Comma;
+ result.tag = .Comma;
self.index += 1;
break;
},
'?' => {
- result.id = .QuestionMark;
+ result.tag = .QuestionMark;
self.index += 1;
break;
},
':' => {
- result.id = .Colon;
+ result.tag = .Colon;
self.index += 1;
break;
},
@@ -519,20 +519,20 @@ pub const Tokenizer = struct {
},
'\\' => {
state = .backslash;
- result.id = .MultilineStringLiteralLine;
+ result.tag = .MultilineStringLiteralLine;
},
'{' => {
- result.id = .LBrace;
+ result.tag = .LBrace;
self.index += 1;
break;
},
'}' => {
- result.id = .RBrace;
+ result.tag = .RBrace;
self.index += 1;
break;
},
'~' => {
- result.id = .Tilde;
+ result.tag = .Tilde;
self.index += 1;
break;
},
@@ -550,14 +550,14 @@ pub const Tokenizer = struct {
},
'0' => {
state = .zero;
- result.id = .IntegerLiteral;
+ result.tag = .IntegerLiteral;
},
'1'...'9' => {
state = .int_literal_dec;
- result.id = .IntegerLiteral;
+ result.tag = .IntegerLiteral;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
self.index += 1;
break;
},
@@ -565,42 +565,42 @@ pub const Tokenizer = struct {
.saw_at_sign => switch (c) {
'"' => {
- result.id = .Identifier;
+ result.tag = .Identifier;
state = .string_literal;
},
else => {
// reinterpret as a builtin
self.index -= 1;
state = .builtin;
- result.id = .Builtin;
+ result.tag = .Builtin;
},
},
.ampersand => switch (c) {
'&' => {
- result.id = .Invalid_ampersands;
+ result.tag = .Invalid_ampersands;
self.index += 1;
break;
},
'=' => {
- result.id = .AmpersandEqual;
+ result.tag = .AmpersandEqual;
self.index += 1;
break;
},
else => {
- result.id = .Ampersand;
+ result.tag = .Ampersand;
break;
},
},
.asterisk => switch (c) {
'=' => {
- result.id = .AsteriskEqual;
+ result.tag = .AsteriskEqual;
self.index += 1;
break;
},
'*' => {
- result.id = .AsteriskAsterisk;
+ result.tag = .AsteriskAsterisk;
self.index += 1;
break;
},
@@ -608,43 +608,43 @@ pub const Tokenizer = struct {
state = .asterisk_percent;
},
else => {
- result.id = .Asterisk;
+ result.tag = .Asterisk;
break;
},
},
.asterisk_percent => switch (c) {
'=' => {
- result.id = .AsteriskPercentEqual;
+ result.tag = .AsteriskPercentEqual;
self.index += 1;
break;
},
else => {
- result.id = .AsteriskPercent;
+ result.tag = .AsteriskPercent;
break;
},
},
.percent => switch (c) {
'=' => {
- result.id = .PercentEqual;
+ result.tag = .PercentEqual;
self.index += 1;
break;
},
else => {
- result.id = .Percent;
+ result.tag = .Percent;
break;
},
},
.plus => switch (c) {
'=' => {
- result.id = .PlusEqual;
+ result.tag = .PlusEqual;
self.index += 1;
break;
},
'+' => {
- result.id = .PlusPlus;
+ result.tag = .PlusPlus;
self.index += 1;
break;
},
@@ -652,31 +652,31 @@ pub const Tokenizer = struct {
state = .plus_percent;
},
else => {
- result.id = .Plus;
+ result.tag = .Plus;
break;
},
},
.plus_percent => switch (c) {
'=' => {
- result.id = .PlusPercentEqual;
+ result.tag = .PlusPercentEqual;
self.index += 1;
break;
},
else => {
- result.id = .PlusPercent;
+ result.tag = .PlusPercent;
break;
},
},
.caret => switch (c) {
'=' => {
- result.id = .CaretEqual;
+ result.tag = .CaretEqual;
self.index += 1;
break;
},
else => {
- result.id = .Caret;
+ result.tag = .Caret;
break;
},
},
@@ -684,8 +684,8 @@ pub const Tokenizer = struct {
.identifier => switch (c) {
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
else => {
- if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
- result.id = id;
+ if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
+ result.tag = tag;
}
break;
},
@@ -724,7 +724,7 @@ pub const Tokenizer = struct {
state = .char_literal_backslash;
},
'\'', 0x80...0xbf, 0xf8...0xff => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
0xc0...0xdf => { // 110xxxxx
@@ -746,7 +746,7 @@ pub const Tokenizer = struct {
.char_literal_backslash => switch (c) {
'\n' => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
'x' => {
@@ -769,7 +769,7 @@ pub const Tokenizer = struct {
}
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -780,7 +780,7 @@ pub const Tokenizer = struct {
seen_escape_digits = 0;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
state = .char_literal_unicode_invalid;
},
},
@@ -791,14 +791,14 @@ pub const Tokenizer = struct {
},
'}' => {
if (seen_escape_digits == 0) {
- result.id = .Invalid;
+ result.tag = .Invalid;
state = .char_literal_unicode_invalid;
} else {
state = .char_literal_end;
}
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
state = .char_literal_unicode_invalid;
},
},
@@ -813,12 +813,12 @@ pub const Tokenizer = struct {
.char_literal_end => switch (c) {
'\'' => {
- result.id = .CharLiteral;
+ result.tag = .CharLiteral;
self.index += 1;
break;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -831,7 +831,7 @@ pub const Tokenizer = struct {
}
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -847,58 +847,58 @@ pub const Tokenizer = struct {
.bang => switch (c) {
'=' => {
- result.id = .BangEqual;
+ result.tag = .BangEqual;
self.index += 1;
break;
},
else => {
- result.id = .Bang;
+ result.tag = .Bang;
break;
},
},
.pipe => switch (c) {
'=' => {
- result.id = .PipeEqual;
+ result.tag = .PipeEqual;
self.index += 1;
break;
},
'|' => {
- result.id = .PipePipe;
+ result.tag = .PipePipe;
self.index += 1;
break;
},
else => {
- result.id = .Pipe;
+ result.tag = .Pipe;
break;
},
},
.equal => switch (c) {
'=' => {
- result.id = .EqualEqual;
+ result.tag = .EqualEqual;
self.index += 1;
break;
},
'>' => {
- result.id = .EqualAngleBracketRight;
+ result.tag = .EqualAngleBracketRight;
self.index += 1;
break;
},
else => {
- result.id = .Equal;
+ result.tag = .Equal;
break;
},
},
.minus => switch (c) {
'>' => {
- result.id = .Arrow;
+ result.tag = .Arrow;
self.index += 1;
break;
},
'=' => {
- result.id = .MinusEqual;
+ result.tag = .MinusEqual;
self.index += 1;
break;
},
@@ -906,19 +906,19 @@ pub const Tokenizer = struct {
state = .minus_percent;
},
else => {
- result.id = .Minus;
+ result.tag = .Minus;
break;
},
},
.minus_percent => switch (c) {
'=' => {
- result.id = .MinusPercentEqual;
+ result.tag = .MinusPercentEqual;
self.index += 1;
break;
},
else => {
- result.id = .MinusPercent;
+ result.tag = .MinusPercent;
break;
},
},
@@ -928,24 +928,24 @@ pub const Tokenizer = struct {
state = .angle_bracket_angle_bracket_left;
},
'=' => {
- result.id = .AngleBracketLeftEqual;
+ result.tag = .AngleBracketLeftEqual;
self.index += 1;
break;
},
else => {
- result.id = .AngleBracketLeft;
+ result.tag = .AngleBracketLeft;
break;
},
},
.angle_bracket_angle_bracket_left => switch (c) {
'=' => {
- result.id = .AngleBracketAngleBracketLeftEqual;
+ result.tag = .AngleBracketAngleBracketLeftEqual;
self.index += 1;
break;
},
else => {
- result.id = .AngleBracketAngleBracketLeft;
+ result.tag = .AngleBracketAngleBracketLeft;
break;
},
},
@@ -955,24 +955,24 @@ pub const Tokenizer = struct {
state = .angle_bracket_angle_bracket_right;
},
'=' => {
- result.id = .AngleBracketRightEqual;
+ result.tag = .AngleBracketRightEqual;
self.index += 1;
break;
},
else => {
- result.id = .AngleBracketRight;
+ result.tag = .AngleBracketRight;
break;
},
},
.angle_bracket_angle_bracket_right => switch (c) {
'=' => {
- result.id = .AngleBracketAngleBracketRightEqual;
+ result.tag = .AngleBracketAngleBracketRightEqual;
self.index += 1;
break;
},
else => {
- result.id = .AngleBracketAngleBracketRight;
+ result.tag = .AngleBracketAngleBracketRight;
break;
},
},
@@ -985,30 +985,30 @@ pub const Tokenizer = struct {
state = .period_asterisk;
},
else => {
- result.id = .Period;
+ result.tag = .Period;
break;
},
},
.period_2 => switch (c) {
'.' => {
- result.id = .Ellipsis3;
+ result.tag = .Ellipsis3;
self.index += 1;
break;
},
else => {
- result.id = .Ellipsis2;
+ result.tag = .Ellipsis2;
break;
},
},
.period_asterisk => switch (c) {
'*' => {
- result.id = .Invalid_periodasterisks;
+ result.tag = .Invalid_periodasterisks;
break;
},
else => {
- result.id = .PeriodAsterisk;
+ result.tag = .PeriodAsterisk;
break;
},
},
@@ -1016,15 +1016,15 @@ pub const Tokenizer = struct {
.slash => switch (c) {
'/' => {
state = .line_comment_start;
- result.id = .LineComment;
+ result.tag = .LineComment;
},
'=' => {
- result.id = .SlashEqual;
+ result.tag = .SlashEqual;
self.index += 1;
break;
},
else => {
- result.id = .Slash;
+ result.tag = .Slash;
break;
},
},
@@ -1033,7 +1033,7 @@ pub const Tokenizer = struct {
state = .doc_comment_start;
},
'!' => {
- result.id = .ContainerDocComment;
+ result.tag = .ContainerDocComment;
state = .container_doc_comment;
},
'\n' => break,
@@ -1048,16 +1048,16 @@ pub const Tokenizer = struct {
state = .line_comment;
},
'\n' => {
- result.id = .DocComment;
+ result.tag = .DocComment;
break;
},
'\t', '\r' => {
state = .doc_comment;
- result.id = .DocComment;
+ result.tag = .DocComment;
},
else => {
state = .doc_comment;
- result.id = .DocComment;
+ result.tag = .DocComment;
self.checkLiteralCharacter();
},
},
@@ -1083,7 +1083,7 @@ pub const Tokenizer = struct {
},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1093,7 +1093,7 @@ pub const Tokenizer = struct {
state = .int_literal_bin;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1104,7 +1104,7 @@ pub const Tokenizer = struct {
'0'...'1' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1114,7 +1114,7 @@ pub const Tokenizer = struct {
state = .int_literal_oct;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1125,7 +1125,7 @@ pub const Tokenizer = struct {
'0'...'7' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1135,7 +1135,7 @@ pub const Tokenizer = struct {
state = .int_literal_dec;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1145,16 +1145,16 @@ pub const Tokenizer = struct {
},
'.' => {
state = .num_dot_dec;
- result.id = .FloatLiteral;
+ result.tag = .FloatLiteral;
},
'e', 'E' => {
state = .float_exponent_unsigned;
- result.id = .FloatLiteral;
+ result.tag = .FloatLiteral;
},
'0'...'9' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1164,7 +1164,7 @@ pub const Tokenizer = struct {
state = .int_literal_hex;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1174,23 +1174,23 @@ pub const Tokenizer = struct {
},
'.' => {
state = .num_dot_hex;
- result.id = .FloatLiteral;
+ result.tag = .FloatLiteral;
},
'p', 'P' => {
state = .float_exponent_unsigned;
- result.id = .FloatLiteral;
+ result.tag = .FloatLiteral;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
},
.num_dot_dec => switch (c) {
'.' => {
- result.id = .IntegerLiteral;
+ result.tag = .IntegerLiteral;
self.index -= 1;
state = .start;
break;
@@ -1203,14 +1203,14 @@ pub const Tokenizer = struct {
},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
},
.num_dot_hex => switch (c) {
'.' => {
- result.id = .IntegerLiteral;
+ result.tag = .IntegerLiteral;
self.index -= 1;
state = .start;
break;
@@ -1219,12 +1219,12 @@ pub const Tokenizer = struct {
state = .float_exponent_unsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {
- result.id = .FloatLiteral;
+ result.tag = .FloatLiteral;
state = .float_fraction_hex;
},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1234,7 +1234,7 @@ pub const Tokenizer = struct {
state = .float_fraction_dec;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1248,7 +1248,7 @@ pub const Tokenizer = struct {
'0'...'9' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1258,7 +1258,7 @@ pub const Tokenizer = struct {
state = .float_fraction_hex;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1272,7 +1272,7 @@ pub const Tokenizer = struct {
'0'...'9', 'a'...'f', 'A'...'F' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1292,7 +1292,7 @@ pub const Tokenizer = struct {
state = .float_exponent_num;
},
else => {
- result.id = .Invalid;
+ result.tag = .Invalid;
break;
},
},
@@ -1303,7 +1303,7 @@ pub const Tokenizer = struct {
'0'...'9' => {},
else => {
if (isIdentifierChar(c)) {
- result.id = .Invalid;
+ result.tag = .Invalid;
}
break;
},
@@ -1327,18 +1327,18 @@ pub const Tokenizer = struct {
=> {},
.identifier => {
- if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
- result.id = id;
+ if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {
+ result.tag = tag;
}
},
.line_comment, .line_comment_start => {
- result.id = .LineComment;
+ result.tag = .LineComment;
},
.doc_comment, .doc_comment_start => {
- result.id = .DocComment;
+ result.tag = .DocComment;
},
.container_doc_comment => {
- result.id = .ContainerDocComment;
+ result.tag = .ContainerDocComment;
},
.int_literal_dec_no_underscore,
@@ -1361,76 +1361,76 @@ pub const Tokenizer = struct {
.char_literal_unicode,
.string_literal_backslash,
=> {
- result.id = .Invalid;
+ result.tag = .Invalid;
},
.equal => {
- result.id = .Equal;
+ result.tag = .Equal;
},
.bang => {
- result.id = .Bang;
+ result.tag = .Bang;
},
.minus => {
- result.id = .Minus;
+ result.tag = .Minus;
},
.slash => {
- result.id = .Slash;
+ result.tag = .Slash;
},
.zero => {
- result.id = .IntegerLiteral;
+ result.tag = .IntegerLiteral;
},
.ampersand => {
- result.id = .Ampersand;
+ result.tag = .Ampersand;
},
.period => {
- result.id = .Period;
+ result.tag = .Period;
},
.period_2 => {
- result.id = .Ellipsis2;
+ result.tag = .Ellipsis2;
},
.period_asterisk => {
- result.id = .PeriodAsterisk;
+ result.tag = .PeriodAsterisk;
},
.pipe => {
- result.id = .Pipe;
+ result.tag = .Pipe;
},
.angle_bracket_angle_bracket_right => {
- result.id = .AngleBracketAngleBracketRight;
+ result.tag = .AngleBracketAngleBracketRight;
},
.angle_bracket_right => {
- result.id = .AngleBracketRight;
+ result.tag = .AngleBracketRight;
},
.angle_bracket_angle_bracket_left => {
- result.id = .AngleBracketAngleBracketLeft;
+ result.tag = .AngleBracketAngleBracketLeft;
},
.angle_bracket_left => {
- result.id = .AngleBracketLeft;
+ result.tag = .AngleBracketLeft;
},
.plus_percent => {
- result.id = .PlusPercent;
+ result.tag = .PlusPercent;
},
.plus => {
- result.id = .Plus;
+ result.tag = .Plus;
},
.percent => {
- result.id = .Percent;
+ result.tag = .Percent;
},
.caret => {
- result.id = .Caret;
+ result.tag = .Caret;
},
.asterisk_percent => {
- result.id = .AsteriskPercent;
+ result.tag = .AsteriskPercent;
},
.asterisk => {
- result.id = .Asterisk;
+ result.tag = .Asterisk;
},
.minus_percent => {
- result.id = .MinusPercent;
+ result.tag = .MinusPercent;
},
}
}
- if (result.id == .Eof) {
+ if (result.tag == .Eof) {
if (self.pending_invalid_token) |token| {
self.pending_invalid_token = null;
return token;
@@ -1446,7 +1446,7 @@ pub const Tokenizer = struct {
const invalid_length = self.getInvalidCharacterLength();
if (invalid_length == 0) return;
self.pending_invalid_token = .{
- .id = .Invalid,
+ .tag = .Invalid,
.loc = .{
.start = self.index,
.end = self.index + invalid_length,
@@ -1493,14 +1493,14 @@ pub const Tokenizer = struct {
};
test "tokenizer" {
- testTokenize("test", &[_]Token.Id{.Keyword_test});
+ testTokenize("test", &[_]Token.Tag{.Keyword_test});
}
test "tokenizer - unknown length pointer and then c pointer" {
testTokenize(
\\[*]u8
\\[*c]u8
- , &[_]Token.Id{
+ , &[_]Token.Tag{
.LBracket,
.Asterisk,
.RBracket,
@@ -1516,70 +1516,70 @@ test "tokenizer - unknown length pointer and then c pointer" {
test "tokenizer - char literal with hex escape" {
testTokenize(
\\'\x1b'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\'\x1'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
}
test "tokenizer - char literal with unicode escapes" {
// Valid unicode escapes
testTokenize(
\\'\u{3}'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\'\u{01}'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\'\u{2a}'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\'\u{3f9}'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\'\u{6E09aBc1523}'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
testTokenize(
\\"\u{440}"
- , &[_]Token.Id{.StringLiteral});
+ , &[_]Token.Tag{.StringLiteral});
// Invalid unicode escapes
testTokenize(
\\'\u'
- , &[_]Token.Id{.Invalid});
+ , &[_]Token.Tag{.Invalid});
testTokenize(
\\'\u{{'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
testTokenize(
\\'\u{}'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
testTokenize(
\\'\u{s}'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
testTokenize(
\\'\u{2z}'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
testTokenize(
\\'\u{4a'
- , &[_]Token.Id{.Invalid});
+ , &[_]Token.Tag{.Invalid});
// Test old-style unicode literals
testTokenize(
\\'\u0333'
- , &[_]Token.Id{ .Invalid, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .Invalid });
testTokenize(
\\'\U0333'
- , &[_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
+ , &[_]Token.Tag{ .Invalid, .IntegerLiteral, .Invalid });
}
test "tokenizer - char literal with unicode code point" {
testTokenize(
\\'💩'
- , &[_]Token.Id{.CharLiteral});
+ , &[_]Token.Tag{.CharLiteral});
}
test "tokenizer - float literal e exponent" {
- testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
+ testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Tag{
.Identifier,
.Equal,
.FloatLiteral,
@@ -1588,7 +1588,7 @@ test "tokenizer - float literal e exponent" {
}
test "tokenizer - float literal p exponent" {
- testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
+ testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Tag{
.Identifier,
.Equal,
.FloatLiteral,
@@ -1597,71 +1597,71 @@ test "tokenizer - float literal p exponent" {
}
test "tokenizer - chars" {
- testTokenize("'c'", &[_]Token.Id{.CharLiteral});
+ testTokenize("'c'", &[_]Token.Tag{.CharLiteral});
}
test "tokenizer - invalid token characters" {
- testTokenize("#", &[_]Token.Id{.Invalid});
- testTokenize("`", &[_]Token.Id{.Invalid});
- testTokenize("'c", &[_]Token.Id{.Invalid});
- testTokenize("'", &[_]Token.Id{.Invalid});
- testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
+ testTokenize("#", &[_]Token.Tag{.Invalid});
+ testTokenize("`", &[_]Token.Tag{.Invalid});
+ testTokenize("'c", &[_]Token.Tag{.Invalid});
+ testTokenize("'", &[_]Token.Tag{.Invalid});
+ testTokenize("''", &[_]Token.Tag{ .Invalid, .Invalid });
}
test "tokenizer - invalid literal/comment characters" {
- testTokenize("\"\x00\"", &[_]Token.Id{
+ testTokenize("\"\x00\"", &[_]Token.Tag{
.StringLiteral,
.Invalid,
});
- testTokenize("//\x00", &[_]Token.Id{
+ testTokenize("//\x00", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\x1f", &[_]Token.Id{
+ testTokenize("//\x1f", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\x7f", &[_]Token.Id{
+ testTokenize("//\x7f", &[_]Token.Tag{
.LineComment,
.Invalid,
});
}
test "tokenizer - utf8" {
- testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
- testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
+ testTokenize("//\xc2\x80", &[_]Token.Tag{.LineComment});
+ testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Tag{.LineComment});
}
test "tokenizer - invalid utf8" {
- testTokenize("//\x80", &[_]Token.Id{
+ testTokenize("//\x80", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xbf", &[_]Token.Id{
+ testTokenize("//\xbf", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xf8", &[_]Token.Id{
+ testTokenize("//\xf8", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xff", &[_]Token.Id{
+ testTokenize("//\xff", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xc2\xc0", &[_]Token.Id{
+ testTokenize("//\xc2\xc0", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xe0", &[_]Token.Id{
+ testTokenize("//\xe0", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xf0", &[_]Token.Id{
+ testTokenize("//\xf0", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
+ testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Tag{
.LineComment,
.Invalid,
});
@@ -1669,28 +1669,28 @@ test "tokenizer - invalid utf8" {
test "tokenizer - illegal unicode codepoints" {
// unicode newline characters.U+0085, U+2028, U+2029
- testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
- testTokenize("//\xc2\x85", &[_]Token.Id{
+ testTokenize("//\xc2\x84", &[_]Token.Tag{.LineComment});
+ testTokenize("//\xc2\x85", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
- testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
- testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
+ testTokenize("//\xc2\x86", &[_]Token.Tag{.LineComment});
+ testTokenize("//\xe2\x80\xa7", &[_]Token.Tag{.LineComment});
+ testTokenize("//\xe2\x80\xa8", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
+ testTokenize("//\xe2\x80\xa9", &[_]Token.Tag{
.LineComment,
.Invalid,
});
- testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
+ testTokenize("//\xe2\x80\xaa", &[_]Token.Tag{.LineComment});
}
test "tokenizer - string identifier and builtin fns" {
testTokenize(
\\const @"if" = @import("std");
- , &[_]Token.Id{
+ , &[_]Token.Tag{
.Keyword_const,
.Identifier,
.Equal,
@@ -1705,7 +1705,7 @@ test "tokenizer - string identifier and builtin fns" {
test "tokenizer - multiline string literal with literal tab" {
testTokenize(
\\\\foo bar
- , &[_]Token.Id{
+ , &[_]Token.Tag{
.MultilineStringLiteralLine,
});
}
@@ -1718,7 +1718,7 @@ test "tokenizer - comments with literal tab" {
\\// foo
\\/// foo
\\/// /foo
- , &[_]Token.Id{
+ , &[_]Token.Tag{
.LineComment,
.ContainerDocComment,
.DocComment,
@@ -1729,21 +1729,21 @@ test "tokenizer - comments with literal tab" {
}
test "tokenizer - pipe and then invalid" {
- testTokenize("||=", &[_]Token.Id{
+ testTokenize("||=", &[_]Token.Tag{
.PipePipe,
.Equal,
});
}
test "tokenizer - line comment and doc comment" {
- testTokenize("//", &[_]Token.Id{.LineComment});
- testTokenize("// a / b", &[_]Token.Id{.LineComment});
- testTokenize("// /", &[_]Token.Id{.LineComment});
- testTokenize("/// a", &[_]Token.Id{.DocComment});
- testTokenize("///", &[_]Token.Id{.DocComment});
- testTokenize("////", &[_]Token.Id{.LineComment});
- testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
- testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
+ testTokenize("//", &[_]Token.Tag{.LineComment});
+ testTokenize("// a / b", &[_]Token.Tag{.LineComment});
+ testTokenize("// /", &[_]Token.Tag{.LineComment});
+ testTokenize("/// a", &[_]Token.Tag{.DocComment});
+ testTokenize("///", &[_]Token.Tag{.DocComment});
+ testTokenize("////", &[_]Token.Tag{.LineComment});
+ testTokenize("//!", &[_]Token.Tag{.ContainerDocComment});
+ testTokenize("//!!", &[_]Token.Tag{.ContainerDocComment});
}
test "tokenizer - line comment followed by identifier" {
@@ -1751,7 +1751,7 @@ test "tokenizer - line comment followed by identifier" {
\\ Unexpected,
\\ // another
\\ Another,
- , &[_]Token.Id{
+ , &[_]Token.Tag{
.Identifier,
.Comma,
.LineComment,
@@ -1761,14 +1761,14 @@ test "tokenizer - line comment followed by identifier" {
}
test "tokenizer - UTF-8 BOM is recognized and skipped" {
- testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
+ testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Tag{
.Identifier,
.Semicolon,
});
}
test "correctly parse pointer assignment" {
- testTokenize("b.*=3;\n", &[_]Token.Id{
+ testTokenize("b.*=3;\n", &[_]Token.Tag{
.Identifier,
.PeriodAsterisk,
.Equal,
@@ -1778,14 +1778,14 @@ test "correctly parse pointer assignment" {
}
test "correctly parse pointer dereference followed by asterisk" {
- testTokenize("\"b\".* ** 10", &[_]Token.Id{
+ testTokenize("\"b\".* ** 10", &[_]Token.Tag{
.StringLiteral,
.PeriodAsterisk,
.AsteriskAsterisk,
.IntegerLiteral,
});
- testTokenize("(\"b\".*)** 10", &[_]Token.Id{
+ testTokenize("(\"b\".*)** 10", &[_]Token.Tag{
.LParen,
.StringLiteral,
.PeriodAsterisk,
@@ -1794,7 +1794,7 @@ test "correctly parse pointer dereference followed by asterisk" {
.IntegerLiteral,
});
- testTokenize("\"b\".*** 10", &[_]Token.Id{
+ testTokenize("\"b\".*** 10", &[_]Token.Tag{
.StringLiteral,
.Invalid_periodasterisks,
.AsteriskAsterisk,
@@ -1803,252 +1803,252 @@ test "correctly parse pointer dereference followed by asterisk" {
}
test "tokenizer - range literals" {
- testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
- testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
- testTokenize("0x00...0x09", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
- testTokenize("0b00...0b11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
- testTokenize("0o00...0o11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+ testTokenize("0...9", &[_]Token.Tag{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+ testTokenize("'0'...'9'", &[_]Token.Tag{ .CharLiteral, .Ellipsis3, .CharLiteral });
+ testTokenize("0x00...0x09", &[_]Token.Tag{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+ testTokenize("0b00...0b11", &[_]Token.Tag{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
+ testTokenize("0o00...0o11", &[_]Token.Tag{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
}
test "tokenizer - number literals decimal" {
- testTokenize("0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("1", &[_]Token.Id{.IntegerLiteral});
- testTokenize("2", &[_]Token.Id{.IntegerLiteral});
- testTokenize("3", &[_]Token.Id{.IntegerLiteral});
- testTokenize("4", &[_]Token.Id{.IntegerLiteral});
- testTokenize("5", &[_]Token.Id{.IntegerLiteral});
- testTokenize("6", &[_]Token.Id{.IntegerLiteral});
- testTokenize("7", &[_]Token.Id{.IntegerLiteral});
- testTokenize("8", &[_]Token.Id{.IntegerLiteral});
- testTokenize("9", &[_]Token.Id{.IntegerLiteral});
- testTokenize("1..", &[_]Token.Id{ .IntegerLiteral, .Ellipsis2 });
- testTokenize("0a", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("9b", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1z", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1z_1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("9z3", &[_]Token.Id{ .Invalid, .Identifier });
-
- testTokenize("0_0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0001", &[_]Token.Id{.IntegerLiteral});
- testTokenize("01234567890", &[_]Token.Id{.IntegerLiteral});
- testTokenize("012_345_6789_0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{.IntegerLiteral});
-
- testTokenize("00_", &[_]Token.Id{.Invalid});
- testTokenize("0_0_", &[_]Token.Id{.Invalid});
- testTokenize("0__0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0_0f", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0_0_f", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0_0_f_00", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1_,", &[_]Token.Id{ .Invalid, .Comma });
-
- testTokenize("1.", &[_]Token.Id{.FloatLiteral});
- testTokenize("0.0", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.0", &[_]Token.Id{.FloatLiteral});
- testTokenize("10.0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0e0", &[_]Token.Id{.FloatLiteral});
- testTokenize("1e0", &[_]Token.Id{.FloatLiteral});
- testTokenize("1e100", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.e100", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.0e100", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral});
- testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral});
- testTokenize("1.+", &[_]Token.Id{ .FloatLiteral, .Plus });
-
- testTokenize("1e", &[_]Token.Id{.Invalid});
- testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0p100", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Identifier, .Minus, .IntegerLiteral });
- testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0_,", &[_]Token.Id{ .Invalid, .Comma });
- testTokenize("1_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
- testTokenize("1._", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.a", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.z", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1._0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1._+", &[_]Token.Id{ .Invalid, .Identifier, .Plus });
- testTokenize("1._e", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0e", &[_]Token.Id{.Invalid});
- testTokenize("1.0e,", &[_]Token.Id{ .Invalid, .Comma });
- testTokenize("1.0e_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0e+_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0e-_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("1.0e0_+", &[_]Token.Id{ .Invalid, .Plus });
+ testTokenize("0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("1", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("2", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("3", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("4", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("5", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("6", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("7", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("8", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("9", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("1..", &[_]Token.Tag{ .IntegerLiteral, .Ellipsis2 });
+ testTokenize("0a", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("9b", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1z", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1z_1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("9z3", &[_]Token.Tag{ .Invalid, .Identifier });
+
+ testTokenize("0_0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0001", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("01234567890", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("012_345_6789_0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Tag{.IntegerLiteral});
+
+ testTokenize("00_", &[_]Token.Tag{.Invalid});
+ testTokenize("0_0_", &[_]Token.Tag{.Invalid});
+ testTokenize("0__0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0_0f", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0_0_f", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0_0_f_00", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1_,", &[_]Token.Tag{ .Invalid, .Comma });
+
+ testTokenize("1.", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0.0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("10.0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0e0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1e0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1e100", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.e100", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.0e100", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.0e+100", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.0e-100", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("1.+", &[_]Token.Tag{ .FloatLiteral, .Plus });
+
+ testTokenize("1e", &[_]Token.Tag{.Invalid});
+ testTokenize("1.0e1f0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0p100", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0p-100", &[_]Token.Tag{ .Invalid, .Identifier, .Minus, .IntegerLiteral });
+ testTokenize("1.0p1f0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0_,", &[_]Token.Tag{ .Invalid, .Comma });
+ testTokenize("1_.0", &[_]Token.Tag{ .Invalid, .Period, .IntegerLiteral });
+ testTokenize("1._", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.a", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.z", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1._0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1._+", &[_]Token.Tag{ .Invalid, .Identifier, .Plus });
+ testTokenize("1._e", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0e", &[_]Token.Tag{.Invalid});
+ testTokenize("1.0e,", &[_]Token.Tag{ .Invalid, .Comma });
+ testTokenize("1.0e_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0e+_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0e-_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("1.0e0_+", &[_]Token.Tag{ .Invalid, .Plus });
}
test "tokenizer - number literals binary" {
- testTokenize("0b0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b1", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b2", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b3", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b4", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b5", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b6", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b7", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0b9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0ba", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0bb", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0bc", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0bd", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0be", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0bf", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0bz", &[_]Token.Id{ .Invalid, .Identifier });
-
- testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b10_10_10_10", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
- testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
-
- testTokenize("0B0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b_0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b1_", &[_]Token.Id{.Invalid});
- testTokenize("0b0__1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b0_1_", &[_]Token.Id{.Invalid});
- testTokenize("0b1e", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b1p", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b1e0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b1p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0b1_,", &[_]Token.Id{ .Invalid, .Comma });
+ testTokenize("0b0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b1", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b2", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b3", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b4", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b5", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b6", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b7", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b8", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0b9", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0ba", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0bb", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0bc", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0bd", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0be", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0bf", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0bz", &[_]Token.Tag{ .Invalid, .Identifier });
+
+ testTokenize("0b0000_0000", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b1111_1111", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b10_10_10_10", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0b1.", &[_]Token.Tag{ .IntegerLiteral, .Period });
+ testTokenize("0b1.0", &[_]Token.Tag{ .IntegerLiteral, .Period, .IntegerLiteral });
+
+ testTokenize("0B0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b_0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0b0__1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b0_1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0b1e", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b1p", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b1e0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b1p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0b1_,", &[_]Token.Tag{ .Invalid, .Comma });
}
test "tokenizer - number literals octal" {
- testTokenize("0o0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o1", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o2", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o3", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o4", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o5", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o6", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o7", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0o9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
- testTokenize("0oa", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0ob", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0oc", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0od", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0oe", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0of", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0oz", &[_]Token.Id{ .Invalid, .Identifier });
-
- testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o01_23_45_67", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
- testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
-
- testTokenize("0O0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o_0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o1_", &[_]Token.Id{.Invalid});
- testTokenize("0o0__1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o0_1_", &[_]Token.Id{.Invalid});
- testTokenize("0o1e", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o1p", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o1e0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o1p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0o_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
+ testTokenize("0o0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o1", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o2", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o3", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o4", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o5", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o6", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o7", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o8", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0o9", &[_]Token.Tag{ .Invalid, .IntegerLiteral });
+ testTokenize("0oa", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0ob", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0oc", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0od", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0oe", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0of", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0oz", &[_]Token.Tag{ .Invalid, .Identifier });
+
+ testTokenize("0o01234567", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o0123_4567", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o01_23_45_67", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0o7.", &[_]Token.Tag{ .IntegerLiteral, .Period });
+ testTokenize("0o7.0", &[_]Token.Tag{ .IntegerLiteral, .Period, .IntegerLiteral });
+
+ testTokenize("0O0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o_0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0o0__1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o0_1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0o1e", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o1p", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o1e0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o1p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0o_,", &[_]Token.Tag{ .Invalid, .Identifier, .Comma });
}
test "tokenizer - number literals hexadeciaml" {
- testTokenize("0x0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x1", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x2", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x3", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x4", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x5", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x6", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x7", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x8", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x9", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xa", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xb", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xc", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xd", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xe", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xf", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xA", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xB", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xC", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xD", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xE", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0xF", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x0z", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0xz", &[_]Token.Id{ .Invalid, .Identifier });
-
- testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral});
-
- testTokenize("0X0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x_", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x_1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x1_", &[_]Token.Id{.Invalid});
- testTokenize("0x0__1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0_1_", &[_]Token.Id{.Invalid});
- testTokenize("0x_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
-
- testTokenize("0x1.", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xF.", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xF.0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xF.F", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xF.Fp0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x1.+0xF.", &[_]Token.Id{ .FloatLiteral, .Plus, .FloatLiteral });
-
- testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x0p0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xff.p10", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xffp10", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{.FloatLiteral});
- testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{.FloatLiteral});
-
- testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral});
- testTokenize("0x1p", &[_]Token.Id{.Invalid});
- testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.p", &[_]Token.Id{.Invalid});
- testTokenize("0x0.z", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0._", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
- testTokenize("0x0_.0.0", &[_]Token.Id{ .Invalid, .Period, .FloatLiteral });
- testTokenize("0x0._0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0_", &[_]Token.Id{.Invalid});
- testTokenize("0x0_p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid, .Period, .Identifier });
- testTokenize("0x0._p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid, .Identifier });
- testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid, .Eof });
+ testTokenize("0x0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x1", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x2", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x3", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x4", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x5", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x6", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x7", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x8", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x9", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xa", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xb", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xc", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xd", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xe", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xf", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xA", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xB", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xC", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xD", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xE", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0xF", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x0z", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0xz", &[_]Token.Tag{ .Invalid, .Identifier });
+
+ testTokenize("0x0123456789ABCDEF", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Tag{.IntegerLiteral});
+
+ testTokenize("0X0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x_", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x_1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0x0__1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0_1_", &[_]Token.Tag{.Invalid});
+ testTokenize("0x_,", &[_]Token.Tag{ .Invalid, .Identifier, .Comma });
+
+ testTokenize("0x1.", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x1.0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xF.", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xF.0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xF.F", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xF.Fp0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xF.FP0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x1p0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xfp0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x1.+0xF.", &[_]Token.Tag{ .FloatLiteral, .Plus, .FloatLiteral });
+
+ testTokenize("0x0123456.789ABCDEF", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x0p0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0x0.0p0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xff.ffp10", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xff.ffP10", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xff.p10", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xffp10", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Tag{.FloatLiteral});
+ testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Tag{.FloatLiteral});
+
+ testTokenize("0x1e", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x1e0", &[_]Token.Tag{.IntegerLiteral});
+ testTokenize("0x1p", &[_]Token.Tag{.Invalid});
+ testTokenize("0xfp0z1", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0xff.ffpff", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.p", &[_]Token.Tag{.Invalid});
+ testTokenize("0x0.z", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0._", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0_.0", &[_]Token.Tag{ .Invalid, .Period, .IntegerLiteral });
+ testTokenize("0x0_.0.0", &[_]Token.Tag{ .Invalid, .Period, .FloatLiteral });
+ testTokenize("0x0._0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0_", &[_]Token.Tag{.Invalid});
+ testTokenize("0x0_p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0_.p0", &[_]Token.Tag{ .Invalid, .Period, .Identifier });
+ testTokenize("0x0._p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0_p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0._0p0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0p_0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0p+_0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0p-_0", &[_]Token.Tag{ .Invalid, .Identifier });
+ testTokenize("0x0.0p0_", &[_]Token.Tag{ .Invalid, .Eof });
}
-fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
+fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {
const token = tokenizer.next();
- if (token.id != expected_token_id) {
- std.debug.panic("expected {s}, found {s}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
+ if (token.tag != expected_token_id) {
+ std.debug.panic("expected {s}, found {s}\n", .{ @tagName(expected_token_id), @tagName(token.tag) });
}
}
const last_token = tokenizer.next();
- std.testing.expect(last_token.id == .Eof);
+ std.testing.expect(last_token.tag == .Eof);
}