Commit 8df0841d6a
Changed files (6)
src-self-hosted
lib/std/zig/ast.zig
@@ -10,7 +10,8 @@ pub const NodeIndex = usize;
pub const Tree = struct {
/// Reference to externally-owned data.
source: []const u8,
- tokens: []const Token,
+ token_ids: []const Token.Id,
+ token_locs: []const Token.Loc,
errors: []const Error,
/// undefined on parse error (when errors field is not empty)
root_node: *Node.Root,
@@ -23,26 +24,27 @@ pub const Tree = struct {
generated: bool = false,
pub fn deinit(self: *Tree) void {
- self.gpa.free(self.tokens);
+ self.gpa.free(self.token_ids);
+ self.gpa.free(self.token_locs);
self.gpa.free(self.errors);
self.arena.promote(self.gpa).deinit();
}
pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void {
- return parse_error.render(self.tokens, stream);
+ return parse_error.render(self.token_ids, stream);
}
pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 {
- return self.tokenSlicePtr(self.tokens[token_index]);
+ return self.tokenSliceLoc(self.token_locs[token_index]);
}
- pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 {
+ pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 {
return self.source[token.start..token.end];
}
pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 {
- const first_token = self.tokens[node.firstToken()];
- const last_token = self.tokens[node.lastToken()];
+ const first_token = self.token_locs[node.firstToken()];
+ const last_token = self.token_locs[node.lastToken()];
return self.source[first_token.start..last_token.end];
}
@@ -54,7 +56,7 @@ pub const Tree = struct {
};
/// Return the Location of the token relative to the offset specified by `start_index`.
- pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location {
+ pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location {
var loc = Location{
.line = 0,
.column = 0,
@@ -82,14 +84,14 @@ pub const Tree = struct {
}
pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location {
- return self.tokenLocationPtr(start_index, self.tokens[token_index]);
+ return self.tokenLocationLoc(start_index, self.token_locs[token_index]);
}
pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool {
- return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]);
+ return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]);
}
- pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool {
+ pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool {
return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null;
}
@@ -100,7 +102,7 @@ pub const Tree = struct {
/// Skips over comments
pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex {
var index = token_index - 1;
- while (self.tokens[index].id == Token.Id.LineComment) {
+ while (self.token_ids[index] == Token.Id.LineComment) {
index -= 1;
}
return index;
@@ -109,7 +111,7 @@ pub const Tree = struct {
/// Skips over comments
pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex {
var index = token_index + 1;
- while (self.tokens[index].id == Token.Id.LineComment) {
+ while (self.token_ids[index] == Token.Id.LineComment) {
index += 1;
}
return index;
@@ -166,7 +168,7 @@ pub const Error = union(enum) {
DeclBetweenFields: DeclBetweenFields,
InvalidAnd: InvalidAnd,
- pub fn render(self: *const Error, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void {
switch (self.*) {
.InvalidToken => |*x| return x.render(tokens, stream),
.ExpectedContainerMembers => |*x| return x.render(tokens, stream),
@@ -321,7 +323,7 @@ pub const Error = union(enum) {
pub const ExpectedCall = struct {
node: *Node,
- pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void {
return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{
@tagName(self.node.id),
});
@@ -331,7 +333,7 @@ pub const Error = union(enum) {
pub const ExpectedCallOrFnProto = struct {
node: *Node,
- pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void {
return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++
@tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)});
}
@@ -341,14 +343,14 @@ pub const Error = union(enum) {
token: TokenIndex,
expected_id: Token.Id,
- pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void {
const found_token = tokens[self.token];
- switch (found_token.id) {
+ switch (found_token) {
.Invalid => {
return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
},
else => {
- const token_name = found_token.id.symbol();
+ const token_name = found_token.symbol();
return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
},
}
@@ -359,11 +361,11 @@ pub const Error = union(enum) {
token: TokenIndex,
end_id: Token.Id,
- pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void {
const actual_token = tokens[self.token];
return stream.print("expected ',' or '{}', found '{}'", .{
self.end_id.symbol(),
- actual_token.id.symbol(),
+ actual_token.symbol(),
});
}
};
@@ -374,9 +376,9 @@ pub const Error = union(enum) {
token: TokenIndex,
- pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
const actual_token = tokens[self.token];
- return stream.print(msg, .{actual_token.id.symbol()});
+ return stream.print(msg, .{actual_token.symbol()});
}
};
}
@@ -387,7 +389,7 @@ pub const Error = union(enum) {
token: TokenIndex,
- pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void {
+ pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void {
return stream.writeAll(msg);
}
};
lib/std/zig/parse.zig
@@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error;
pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
// TODO optimization idea: ensureCapacity on the tokens list and
// then appendAssumeCapacity inside the loop.
- var tokens = std.ArrayList(Token).init(gpa);
- defer tokens.deinit();
+ var token_ids = std.ArrayList(Token.Id).init(gpa);
+ defer token_ids.deinit();
+ var token_locs = std.ArrayList(Token.Loc).init(gpa);
+ defer token_locs.deinit();
var tokenizer = std.zig.Tokenizer.init(source);
while (true) {
- const tree_token = try tokens.addOne();
- tree_token.* = tokenizer.next();
- if (tree_token.id == .Eof) break;
+ const token = tokenizer.next();
+ try token_ids.append(token.id);
+ try token_locs.append(token.loc);
+ if (token.id == .Eof) break;
}
var parser: Parser = .{
.source = source,
.arena = std.heap.ArenaAllocator.init(gpa),
.gpa = gpa,
- .tokens = tokens.items,
+ .token_ids = token_ids.items,
+ .token_locs = token_locs.items,
.errors = .{},
.tok_i = 0,
};
defer parser.errors.deinit(gpa);
errdefer parser.arena.deinit();
- while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1;
+ while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1;
const root_node = try parser.parseRoot();
@@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
tree.* = .{
.gpa = gpa,
.source = source,
- .tokens = tokens.toOwnedSlice(),
+ .token_ids = token_ids.toOwnedSlice(),
+ .token_locs = token_locs.toOwnedSlice(),
.errors = parser.errors.toOwnedSlice(gpa),
.root_node = root_node,
.arena = parser.arena.state,
@@ -58,9 +63,8 @@ const Parser = struct {
arena: std.heap.ArenaAllocator,
gpa: *Allocator,
source: []const u8,
- /// TODO: Optimization idea: have this be several arrays of the token fields rather
- /// than an array of structs.
- tokens: []const Token,
+ token_ids: []const Token.Id,
+ token_locs: []const Token.Loc,
tok_i: TokenIndex,
errors: std.ArrayListUnmanaged(AstError),
@@ -80,19 +84,6 @@ const Parser = struct {
return node;
}
- /// Helper function for appending elements to a singly linked list.
- fn llpush(
- p: *Parser,
- comptime T: type,
- it: *?*std.SinglyLinkedList(T).Node,
- data: T,
- ) !*?*std.SinglyLinkedList(T).Node {
- const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node);
- llnode.* = .{ .data = data };
- it.* = llnode;
- return &llnode.next;
- }
-
/// ContainerMembers
/// <- TestDecl ContainerMembers
/// / TopLevelComptime ContainerMembers
@@ -228,7 +219,7 @@ const Parser = struct {
// try to continue parsing
const index = p.tok_i;
p.findNextContainerMember();
- const next = p.tokens[p.tok_i].id;
+ const next = p.token_ids[p.tok_i];
switch (next) {
.Eof => break,
else => {
@@ -257,7 +248,7 @@ const Parser = struct {
});
}
- const next = p.tokens[p.tok_i].id;
+ const next = p.token_ids[p.tok_i];
switch (next) {
.Eof => break,
.Keyword_comptime => {
@@ -291,7 +282,7 @@ const Parser = struct {
var level: u32 = 0;
while (true) {
const tok = p.nextToken();
- switch (tok.ptr.id) {
+ switch (p.token_ids[tok]) {
// any of these can start a new top level declaration
.Keyword_test,
.Keyword_comptime,
@@ -308,7 +299,7 @@ const Parser = struct {
.Identifier,
=> {
if (level == 0) {
- p.putBackToken(tok.index);
+ p.putBackToken(tok);
return;
}
},
@@ -325,13 +316,13 @@ const Parser = struct {
.RBrace => {
if (level == 0) {
// end of container, exit
- p.putBackToken(tok.index);
+ p.putBackToken(tok);
return;
}
level -= 1;
},
.Eof => {
- p.putBackToken(tok.index);
+ p.putBackToken(tok);
return;
},
else => {},
@@ -344,11 +335,11 @@ const Parser = struct {
var level: u32 = 0;
while (true) {
const tok = p.nextToken();
- switch (tok.ptr.id) {
+ switch (p.token_ids[tok]) {
.LBrace => level += 1,
.RBrace => {
if (level == 0) {
- p.putBackToken(tok.index);
+ p.putBackToken(tok);
return;
}
level -= 1;
@@ -359,7 +350,7 @@ const Parser = struct {
}
},
.Eof => {
- p.putBackToken(tok.index);
+ p.putBackToken(tok);
return;
},
else => {},
@@ -454,8 +445,8 @@ const Parser = struct {
}
if (extern_export_inline_token) |token| {
- if (p.tokens[token].id == .Keyword_inline or
- p.tokens[token].id == .Keyword_noinline)
+ if (p.token_ids[token] == .Keyword_inline or
+ p.token_ids[token] == .Keyword_noinline)
{
try p.errors.append(p.gpa, .{
.ExpectedFn = .{ .token = p.tok_i },
@@ -722,7 +713,7 @@ const Parser = struct {
const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer);
if (defer_token) |token| {
- const payload = if (p.tokens[token].id == .Keyword_errdefer)
+ const payload = if (p.token_ids[token] == .Keyword_errdefer)
try p.parsePayload()
else
null;
@@ -2269,7 +2260,7 @@ const Parser = struct {
/// / EQUAL
fn parseAssignOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.AsteriskEqual => .AssignMul,
.SlashEqual => .AssignDiv,
.PercentEqual => .AssignMod,
@@ -2285,14 +2276,14 @@ const Parser = struct {
.MinusPercentEqual => .AssignSubWrap,
.Equal => .Assign,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
const node = try p.arena.allocator.create(Node.InfixOp);
node.* = .{
- .op_token = token.index,
+ .op_token = token,
.lhs = undefined, // set by caller
.op = op,
.rhs = undefined, // set by caller
@@ -2309,7 +2300,7 @@ const Parser = struct {
/// / RARROWEQUAL
fn parseCompareOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.EqualEqual => .EqualEqual,
.BangEqual => .BangEqual,
.AngleBracketLeft => .LessThan,
@@ -2317,12 +2308,12 @@ const Parser = struct {
.AngleBracketLeftEqual => .LessOrEqual,
.AngleBracketRightEqual => .GreaterOrEqual,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
- return p.createInfixOp(token.index, op);
+ return p.createInfixOp(token, op);
}
/// BitwiseOp
@@ -2333,19 +2324,19 @@ const Parser = struct {
/// / KEYWORD_catch Payload?
fn parseBitwiseOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.Ampersand => .BitAnd,
.Caret => .BitXor,
.Pipe => .BitOr,
.Keyword_orelse => .UnwrapOptional,
.Keyword_catch => .{ .Catch = try p.parsePayload() },
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
- return p.createInfixOp(token.index, op);
+ return p.createInfixOp(token, op);
}
/// BitShiftOp
@@ -2353,16 +2344,16 @@ const Parser = struct {
/// / RARROW2
fn parseBitShiftOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.AngleBracketAngleBracketLeft => .BitShiftLeft,
.AngleBracketAngleBracketRight => .BitShiftRight,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
- return p.createInfixOp(token.index, op);
+ return p.createInfixOp(token, op);
}
/// AdditionOp
@@ -2373,19 +2364,19 @@ const Parser = struct {
/// / MINUSPERCENT
fn parseAdditionOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.Plus => .Add,
.Minus => .Sub,
.PlusPlus => .ArrayCat,
.PlusPercent => .AddWrap,
.MinusPercent => .SubWrap,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
- return p.createInfixOp(token.index, op);
+ return p.createInfixOp(token, op);
}
/// MultiplyOp
@@ -2397,7 +2388,7 @@ const Parser = struct {
/// / ASTERISKPERCENT
fn parseMultiplyOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.InfixOp.Op = switch (token.ptr.id) {
+ const op: Node.InfixOp.Op = switch (p.token_ids[token]) {
.PipePipe => .MergeErrorSets,
.Asterisk => .Mul,
.Slash => .Div,
@@ -2405,12 +2396,12 @@ const Parser = struct {
.AsteriskAsterisk => .ArrayMult,
.AsteriskPercent => .MulWrap,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
- return p.createInfixOp(token.index, op);
+ return p.createInfixOp(token, op);
}
/// PrefixOp
@@ -2423,7 +2414,7 @@ const Parser = struct {
/// / KEYWORD_await
fn parsePrefixOp(p: *Parser) !?*Node {
const token = p.nextToken();
- const op: Node.PrefixOp.Op = switch (token.ptr.id) {
+ const op: Node.PrefixOp.Op = switch (p.token_ids[token]) {
.Bang => .BoolNot,
.Minus => .Negation,
.Tilde => .BitNot,
@@ -2432,14 +2423,14 @@ const Parser = struct {
.Keyword_try => .Try,
.Keyword_await => .Await,
else => {
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
},
};
const node = try p.arena.allocator.create(Node.PrefixOp);
node.* = .{
- .op_token = token.index,
+ .op_token = token,
.op = op,
.rhs = undefined, // set by caller
};
@@ -2493,7 +2484,7 @@ const Parser = struct {
// If the token encountered was **, there will be two nodes instead of one.
// The attributes should be applied to the rightmost operator.
const prefix_op = node.cast(Node.PrefixOp).?;
- var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk)
+ var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk)
&prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType
else
&prefix_op.op.PtrType;
@@ -2812,7 +2803,8 @@ const Parser = struct {
return null;
};
if (p.eatToken(.Identifier)) |ident| {
- const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end];
+ const token_loc = p.token_locs[ident];
+ const token_slice = p.source[token_loc.start..token_loc.end];
if (!std.mem.eql(u8, token_slice, "c")) {
p.putBackToken(ident);
} else {
@@ -2879,7 +2871,7 @@ const Parser = struct {
fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
const kind_token = p.nextToken();
- const init_arg_expr = switch (kind_token.ptr.id) {
+ const init_arg_expr = switch (p.token_ids[kind_token]) {
.Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} },
.Keyword_enum => blk: {
if (p.eatToken(.LParen) != null) {
@@ -2914,13 +2906,13 @@ const Parser = struct {
break :blk Node.ContainerDecl.InitArg{ .None = {} };
},
else => {
- p.putBackToken(kind_token.index);
+ p.putBackToken(kind_token);
return null;
},
};
return ContainerDeclType{
- .kind_token = kind_token.index,
+ .kind_token = kind_token,
.init_arg_expr = init_arg_expr,
};
}
@@ -2973,7 +2965,7 @@ const Parser = struct {
while (try nodeParseFn(p)) |item| {
try list.append(item);
- switch (p.tokens[p.tok_i].id) {
+ switch (p.token_ids[p.tok_i]) {
.Comma => _ = p.nextToken(),
// all possible delimiters
.Colon, .RParen, .RBrace, .RBracket => break,
@@ -2994,13 +2986,13 @@ const Parser = struct {
fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
return struct {
pub fn parse(p: *Parser) Error!?*Node {
- const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) {
- .Keyword_and => p.nextToken().index,
+ const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) {
+ .Keyword_and => p.nextToken(),
.Invalid_ampersands => blk: {
try p.errors.append(p.gpa, .{
.InvalidAnd = .{ .token = p.tok_i },
});
- break :blk p.nextToken().index;
+ break :blk p.nextToken();
},
else => return null,
} else p.eatToken(token) orelse return null;
@@ -3104,7 +3096,7 @@ const Parser = struct {
var tok_i = start_tok_i;
var count: usize = 1; // including first_line
while (true) : (tok_i += 1) {
- switch (p.tokens[tok_i].id) {
+ switch (p.token_ids[tok_i]) {
.LineComment => continue,
.MultilineStringLiteralLine => count += 1,
else => break,
@@ -3118,7 +3110,7 @@ const Parser = struct {
lines[0] = first_line;
count = 1;
while (true) : (tok_i += 1) {
- switch (p.tokens[tok_i].id) {
+ switch (p.token_ids[tok_i]) {
.LineComment => continue,
.MultilineStringLiteralLine => {
lines[count] = tok_i;
@@ -3215,7 +3207,7 @@ const Parser = struct {
}
fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
- return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null;
+ return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null;
}
/// Eat a single-line doc comment on the same line as another node
@@ -3239,7 +3231,7 @@ const Parser = struct {
.PrefixOp => {
var prefix_op = rightmost_op.cast(Node.PrefixOp).?;
// If the token encountered was **, there will be two nodes
- if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) {
+ if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) {
rightmost_op = prefix_op.rhs;
prefix_op = rightmost_op.cast(Node.PrefixOp).?;
}
@@ -3328,11 +3320,7 @@ const Parser = struct {
}
fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex {
- return if (p.eatAnnotatedToken(id)) |token| token.index else null;
- }
-
- fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken {
- return if (p.tokens[p.tok_i].id == id) p.nextToken() else null;
+ return if (p.token_ids[p.tok_i] == id) p.nextToken() else null;
}
fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex {
@@ -3341,29 +3329,25 @@ const Parser = struct {
fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex {
const token = p.nextToken();
- if (token.ptr.id != id) {
+ if (p.token_ids[token] != id) {
try p.errors.append(p.gpa, .{
- .ExpectedToken = .{ .token = token.index, .expected_id = id },
+ .ExpectedToken = .{ .token = token, .expected_id = id },
});
// go back so that we can recover properly
- p.putBackToken(token.index);
+ p.putBackToken(token);
return null;
}
- return token.index;
+ return token;
}
- fn nextToken(p: *Parser) AnnotatedToken {
- const result = AnnotatedToken{
- .index = p.tok_i,
- .ptr = &p.tokens[p.tok_i],
- };
+ fn nextToken(p: *Parser) TokenIndex {
+ const result = p.tok_i;
p.tok_i += 1;
- assert(result.ptr.id != .LineComment);
- if (p.tok_i >= p.tokens.len) return result;
+ assert(p.token_ids[result] != .LineComment);
+ if (p.tok_i >= p.token_ids.len) return result;
while (true) {
- const next_tok = p.tokens[p.tok_i];
- if (next_tok.id != .LineComment) return result;
+ if (p.token_ids[p.tok_i] != .LineComment) return result;
p.tok_i += 1;
}
}
@@ -3371,18 +3355,12 @@ const Parser = struct {
fn putBackToken(p: *Parser, putting_back: TokenIndex) void {
while (p.tok_i > 0) {
p.tok_i -= 1;
- const prev_tok = p.tokens[p.tok_i];
- if (prev_tok.id == .LineComment) continue;
+ if (p.token_ids[p.tok_i] == .LineComment) continue;
assert(putting_back == p.tok_i);
return;
}
}
- const AnnotatedToken = struct {
- index: TokenIndex,
- ptr: *const Token,
- };
-
fn expectNode(
p: *Parser,
parseFn: NodeParseFn,
lib/std/zig/parser_test.zig
@@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b
defer tree.deinit();
for (tree.errors) |*parse_error| {
- const token = tree.tokens[parse_error.loc()];
+ const token = tree.token_locs[parse_error.loc()];
const loc = tree.tokenLocation(0, parse_error.loc());
try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 });
try tree.renderError(parse_error, stderr);
lib/std/zig/render.zig
@@ -68,11 +68,12 @@ fn renderRoot(
tree: *ast.Tree,
) (@TypeOf(stream).Error || Error)!void {
// render all the line comments at the beginning of the file
- for (tree.tokens) |token, i| {
- if (token.id != .LineComment) break;
- try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")});
- const next_token = &tree.tokens[i + 1];
- const loc = tree.tokenLocationPtr(token.end, next_token.*);
+ for (tree.token_ids) |token_id, i| {
+ if (token_id != .LineComment) break;
+ const token_loc = tree.token_locs[i];
+ try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")});
+ const next_token = tree.token_locs[i + 1];
+ const loc = tree.tokenLocationLoc(token_loc.end, next_token);
if (loc.line >= 2) {
try stream.writeByte('\n');
}
@@ -101,8 +102,8 @@ fn renderRoot(
while (token_index != 0) {
token_index -= 1;
- const token = tree.tokens[token_index];
- switch (token.id) {
+ const token_id = tree.token_ids[token_index];
+ switch (token_id) {
.LineComment => {},
.DocComment => {
copy_start_token_index = token_index;
@@ -111,12 +112,13 @@ fn renderRoot(
else => break,
}
- if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
+ const token_loc = tree.token_locs[token_index];
+ if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
if (!found_fmt_directive) {
fmt_active = false;
found_fmt_directive = true;
}
- } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
+ } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
if (!found_fmt_directive) {
fmt_active = true;
found_fmt_directive = true;
@@ -135,7 +137,7 @@ fn renderRoot(
if (decl_i >= root_decls.len) {
// If there's no next reformatted `decl`, just copy the
// remaining input tokens and bail out.
- const start = tree.tokens[copy_start_token_index].start;
+ const start = tree.token_locs[copy_start_token_index].start;
try copyFixingWhitespace(stream, tree.source[start..]);
return;
}
@@ -143,15 +145,16 @@ fn renderRoot(
var decl_first_token_index = decl.firstToken();
while (token_index < decl_first_token_index) : (token_index += 1) {
- const token = tree.tokens[token_index];
- switch (token.id) {
+ const token_id = tree.token_ids[token_index];
+ switch (token_id) {
.LineComment => {},
.Eof => unreachable,
else => continue,
}
- if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) {
+ const token_loc = tree.token_locs[token_index];
+ if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) {
fmt_active = true;
- } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) {
+ } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) {
fmt_active = false;
}
}
@@ -163,8 +166,8 @@ fn renderRoot(
token_index = copy_end_token_index;
while (token_index != 0) {
token_index -= 1;
- const token = tree.tokens[token_index];
- switch (token.id) {
+ const token_id = tree.token_ids[token_index];
+ switch (token_id) {
.LineComment => {},
.DocComment => {
copy_end_token_index = token_index;
@@ -174,8 +177,8 @@ fn renderRoot(
}
}
- const start = tree.tokens[copy_start_token_index].start;
- const end = tree.tokens[copy_end_token_index].start;
+ const start = tree.token_locs[copy_start_token_index].start;
+ const end = tree.token_locs[copy_end_token_index].start;
try copyFixingWhitespace(stream, tree.source[start..end]);
}
@@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs
var prev_token = first_token;
if (prev_token == 0) return;
var newline_threshold: usize = 2;
- while (tree.tokens[prev_token - 1].id == .DocComment) {
- if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) {
+ while (tree.token_ids[prev_token - 1] == .DocComment) {
+ if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) {
newline_threshold += 1;
}
prev_token -= 1;
}
- const prev_token_end = tree.tokens[prev_token - 1].end;
+ const prev_token_end = tree.token_locs[prev_token - 1].end;
const loc = tree.tokenLocation(prev_token_end, first_token);
if (loc.line >= newline_threshold) {
try stream.writeByte('\n');
@@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
const src_has_trailing_comma = blk: {
const maybe_comma = tree.nextToken(field.lastToken());
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
// The trailing comma is emitted at the end, but if it's not present
@@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
.DocComment => {
const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
- const kind = tree.tokens[comment.first_line].id;
+ const kind = tree.token_ids[comment.first_line];
try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
var tok_i = comment.first_line + 1;
while (true) : (tok_i += 1) {
- const tok_id = tree.tokens[tok_i].id;
+ const tok_id = tree.token_ids[tok_i];
if (tok_id == kind) {
try stream.writeByteNTimes(' ', indent);
try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
@@ -436,13 +439,13 @@ fn renderExpression(
try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space);
const after_op_space = blk: {
- const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
+ const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token));
break :blk if (loc.line == 0) op_space else Space.Newline;
};
try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space);
if (after_op_space == Space.Newline and
- tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine)
+ tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine)
{
try stream.writeByteNTimes(' ', indent + indent_delta);
start_col.* = indent + indent_delta;
@@ -463,10 +466,10 @@ fn renderExpression(
switch (prefix_op_node.op) {
.PtrType => |ptr_info| {
- const op_tok_id = tree.tokens[prefix_op_node.op_token].id;
+ const op_tok_id = tree.token_ids[prefix_op_node.op_token];
switch (op_tok_id) {
.Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
- .LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier)
+ .LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier)
try stream.writeAll("[*c")
else
try stream.writeAll("[*"),
@@ -578,8 +581,8 @@ fn renderExpression(
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
- const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
- const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
+ const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
+ const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
const new_indent = if (ends_with_comment) indent + indent_delta else indent;
const new_space = if (ends_with_comment) Space.Newline else Space.None;
try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space);
@@ -653,7 +656,7 @@ fn renderExpression(
return renderToken(tree, stream, rtoken, indent, start_col, space);
}
- if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) {
+ if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) {
const expr = exprs[0];
switch (lhs) {
.dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None),
@@ -675,17 +678,17 @@ fn renderExpression(
for (exprs) |expr, i| {
if (i + 1 < exprs.len) {
const expr_last_token = expr.lastToken() + 1;
- const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken());
+ const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken());
if (loc.line != 0) break :blk count;
count += 1;
} else {
const expr_last_token = expr.lastToken();
- const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken);
+ const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken);
if (loc.line == 0) {
// all on one line
const src_has_trailing_comma = trailblk: {
const maybe_comma = tree.prevToken(rtoken);
- break :trailblk tree.tokens[maybe_comma].id == .Comma;
+ break :trailblk tree.token_ids[maybe_comma] == .Comma;
};
if (src_has_trailing_comma) {
break :blk 1; // force row size 1
@@ -723,7 +726,7 @@ fn renderExpression(
var new_indent = indent + indent_delta;
- if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) {
+ if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) {
try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline);
try stream.writeByteNTimes(' ', new_indent);
} else {
@@ -750,7 +753,7 @@ fn renderExpression(
}
col = 1;
- if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) {
+ if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) {
try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // ,
} else {
try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // ,
@@ -819,11 +822,11 @@ fn renderExpression(
const src_has_trailing_comma = blk: {
const maybe_comma = tree.prevToken(rtoken);
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
const src_same_line = blk: {
- const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken);
+ const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken);
break :blk loc.line == 0;
};
@@ -929,7 +932,7 @@ fn renderExpression(
const src_has_trailing_comma = blk: {
const maybe_comma = tree.prevToken(call.rtoken);
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
if (src_has_trailing_comma) {
@@ -983,8 +986,8 @@ fn renderExpression(
try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [
- const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment;
- const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment;
+ const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment;
+ const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment;
const new_indent = if (ends_with_comment) indent + indent_delta else indent;
const new_space = if (ends_with_comment) Space.Newline else Space.None;
try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space);
@@ -1226,9 +1229,9 @@ fn renderExpression(
var maybe_comma = tree.prevToken(container_decl.lastToken());
// Doc comments for a field may also appear after the comma, eg.
// field_name: T, // comment attached to field_name
- if (tree.tokens[maybe_comma].id == .DocComment)
+ if (tree.token_ids[maybe_comma] == .DocComment)
maybe_comma = tree.prevToken(maybe_comma);
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
const fields_and_decls = container_decl.fieldsAndDecls();
@@ -1321,7 +1324,7 @@ fn renderExpression(
const src_has_trailing_comma = blk: {
const maybe_comma = tree.prevToken(err_set_decl.rbrace_token);
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
if (src_has_trailing_comma) {
@@ -1353,7 +1356,7 @@ fn renderExpression(
try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None);
const comma_token = tree.nextToken(node.lastToken());
- assert(tree.tokens[comma_token].id == .Comma);
+ assert(tree.token_ids[comma_token] == .Comma);
try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // ,
try renderExtraNewline(tree, stream, start_col, decls[i + 1]);
} else {
@@ -1378,7 +1381,7 @@ fn renderExpression(
const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base);
var skip_first_indent = true;
- if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) {
+ if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) {
try stream.print("\n", .{});
skip_first_indent = false;
}
@@ -1406,7 +1409,7 @@ fn renderExpression(
if (builtin_call.params_len < 2) break :blk false;
const last_node = builtin_call.params()[builtin_call.params_len - 1];
const maybe_comma = tree.nextToken(last_node.lastToken());
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
const lparen = tree.nextToken(builtin_call.builtin_token);
@@ -1443,8 +1446,8 @@ fn renderExpression(
const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base);
if (fn_proto.visib_token) |visib_token_index| {
- const visib_token = tree.tokens[visib_token_index];
- assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export);
+ const visib_token = tree.token_ids[visib_token_index];
+ assert(visib_token == .Keyword_pub or visib_token == .Keyword_export);
try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub
}
@@ -1466,7 +1469,7 @@ fn renderExpression(
try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn
break :blk tree.nextToken(fn_proto.fn_token);
};
- assert(tree.tokens[lparen].id == .LParen);
+ assert(tree.token_ids[lparen] == .LParen);
const rparen = tree.prevToken(
// the first token for the annotation expressions is the left
@@ -1482,10 +1485,10 @@ fn renderExpression(
.InferErrorSet => |node| tree.prevToken(node.firstToken()),
.Invalid => unreachable,
});
- assert(tree.tokens[rparen].id == .RParen);
+ assert(tree.token_ids[rparen] == .RParen);
const src_params_trailing_comma = blk: {
- const maybe_comma = tree.tokens[rparen - 1].id;
+ const maybe_comma = tree.token_ids[rparen - 1];
break :blk maybe_comma == .Comma or maybe_comma == .LineComment;
};
@@ -1622,7 +1625,7 @@ fn renderExpression(
const src_has_trailing_comma = blk: {
const last_node = switch_case.items()[switch_case.items_len - 1];
const maybe_comma = tree.nextToken(last_node.lastToken());
- break :blk tree.tokens[maybe_comma].id == .Comma;
+ break :blk tree.token_ids[maybe_comma] == .Comma;
};
if (switch_case.items_len == 1 or !src_has_trailing_comma) {
@@ -1967,7 +1970,7 @@ fn renderExpression(
try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline);
try stream.writeByteNTimes(' ', indent_once);
const comma_or_colon = tree.nextToken(asm_output.lastToken());
- break :blk switch (tree.tokens[comma_or_colon].id) {
+ break :blk switch (tree.token_ids[comma_or_colon]) {
.Comma => tree.nextToken(comma_or_colon),
else => comma_or_colon,
};
@@ -2002,7 +2005,7 @@ fn renderExpression(
try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline);
try stream.writeByteNTimes(' ', indent_once);
const comma_or_colon = tree.nextToken(asm_input.lastToken());
- break :blk switch (tree.tokens[comma_or_colon].id) {
+ break :blk switch (tree.token_ids[comma_or_colon]) {
.Comma => tree.nextToken(comma_or_colon),
else => comma_or_colon,
};
@@ -2205,7 +2208,7 @@ fn renderStatement(
try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None);
const semicolon_index = tree.nextToken(base.lastToken());
- assert(tree.tokens[semicolon_index].id == .Semicolon);
+ assert(tree.token_ids[semicolon_index] == .Semicolon);
try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline);
} else {
try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline);
@@ -2243,22 +2246,25 @@ fn renderTokenOffset(
return;
}
- var token = tree.tokens[token_index];
- try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));
+ var token_loc = tree.token_locs[token_index];
+ try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " "));
if (space == Space.NoComment)
return;
- var next_token = tree.tokens[token_index + 1];
+ var next_token_id = tree.token_ids[token_index + 1];
+ var next_token_loc = tree.token_locs[token_index + 1];
- if (space == Space.Comma) switch (next_token.id) {
+ if (space == Space.Comma) switch (next_token_id) {
.Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),
.LineComment => {
try stream.writeAll(", ");
return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);
},
else => {
- if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) {
+ if (token_index + 2 < tree.token_ids.len and
+ tree.token_ids[token_index + 2] == .MultilineStringLiteralLine)
+ {
try stream.writeAll(",");
return;
} else {
@@ -2271,19 +2277,20 @@ fn renderTokenOffset(
// Skip over same line doc comments
var offset: usize = 1;
- if (next_token.id == .DocComment) {
- const loc = tree.tokenLocationPtr(token.end, next_token);
+ if (next_token_id == .DocComment) {
+ const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
if (loc.line == 0) {
offset += 1;
- next_token = tree.tokens[token_index + offset];
+ next_token_id = tree.token_ids[token_index + offset];
+ next_token_loc = tree.token_locs[token_index + offset];
}
}
- if (next_token.id != .LineComment) blk: {
+ if (next_token_id != .LineComment) blk: {
switch (space) {
Space.None, Space.NoNewline => return,
Space.Newline => {
- if (next_token.id == .MultilineStringLiteralLine) {
+ if (next_token_id == .MultilineStringLiteralLine) {
return;
} else {
try stream.writeAll("\n");
@@ -2292,7 +2299,7 @@ fn renderTokenOffset(
}
},
Space.Space, Space.SpaceOrOutdent => {
- if (next_token.id == .MultilineStringLiteralLine)
+ if (next_token_id == .MultilineStringLiteralLine)
return;
try stream.writeByte(' ');
return;
@@ -2302,14 +2309,15 @@ fn renderTokenOffset(
}
while (true) {
- const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2;
+ const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2;
if (comment_is_empty) {
switch (space) {
Space.Newline => {
offset += 1;
- token = next_token;
- next_token = tree.tokens[token_index + offset];
- if (next_token.id != .LineComment) {
+ token_loc = next_token_loc;
+ next_token_id = tree.token_ids[token_index + offset];
+ next_token_loc = tree.token_locs[token_index + offset];
+ if (next_token_id != .LineComment) {
try stream.writeByte('\n');
start_col.* = 0;
return;
@@ -2322,18 +2330,19 @@ fn renderTokenOffset(
}
}
- var loc = tree.tokenLocationPtr(token.end, next_token);
+ var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
if (loc.line == 0) {
- try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")});
+ try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")});
offset = 2;
- token = next_token;
- next_token = tree.tokens[token_index + offset];
- if (next_token.id != .LineComment) {
+ token_loc = next_token_loc;
+ next_token_loc = tree.token_locs[token_index + offset];
+ next_token_id = tree.token_ids[token_index + offset];
+ if (next_token_id != .LineComment) {
switch (space) {
Space.None, Space.Space => {
try stream.writeByte('\n');
- const after_comment_token = tree.tokens[token_index + offset];
- const next_line_indent = switch (after_comment_token.id) {
+ const after_comment_token = tree.token_ids[token_index + offset];
+ const next_line_indent = switch (after_comment_token) {
.RParen, .RBrace, .RBracket => indent,
else => indent + indent_delta,
};
@@ -2346,7 +2355,7 @@ fn renderTokenOffset(
start_col.* = indent;
},
Space.Newline => {
- if (next_token.id == .MultilineStringLiteralLine) {
+ if (next_token_id == .MultilineStringLiteralLine) {
return;
} else {
try stream.writeAll("\n");
@@ -2359,7 +2368,7 @@ fn renderTokenOffset(
}
return;
}
- loc = tree.tokenLocationPtr(token.end, next_token);
+ loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
}
while (true) {
@@ -2369,15 +2378,16 @@ fn renderTokenOffset(
const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2);
try stream.writeByteNTimes('\n', newline_count);
try stream.writeByteNTimes(' ', indent);
- try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));
+ try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " "));
offset += 1;
- token = next_token;
- next_token = tree.tokens[token_index + offset];
- if (next_token.id != .LineComment) {
+ token_loc = next_token_loc;
+ next_token_loc = tree.token_locs[token_index + offset];
+ next_token_id = tree.token_ids[token_index + offset];
+ if (next_token_id != .LineComment) {
switch (space) {
Space.Newline => {
- if (next_token.id == .MultilineStringLiteralLine) {
+ if (next_token_id == .MultilineStringLiteralLine) {
return;
} else {
try stream.writeAll("\n");
@@ -2388,8 +2398,8 @@ fn renderTokenOffset(
Space.None, Space.Space => {
try stream.writeByte('\n');
- const after_comment_token = tree.tokens[token_index + offset];
- const next_line_indent = switch (after_comment_token.id) {
+ const after_comment_token = tree.token_ids[token_index + offset];
+ const next_line_indent = switch (after_comment_token) {
.RParen, .RBrace, .RBracket => blk: {
if (indent > indent_delta) {
break :blk indent - indent_delta;
@@ -2412,7 +2422,7 @@ fn renderTokenOffset(
}
return;
}
- loc = tree.tokenLocationPtr(token.end, next_token);
+ loc = tree.tokenLocationLoc(token_loc.end, next_token_loc);
}
}
@@ -2448,7 +2458,7 @@ fn renderDocCommentsToken(
) (@TypeOf(stream).Error || Error)!void {
var tok_i = comment.first_line;
while (true) : (tok_i += 1) {
- switch (tree.tokens[tok_i].id) {
+ switch (tree.token_ids[tok_i]) {
.DocComment, .ContainerDocComment => {
if (comment.first_line < first_token) {
try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);
lib/std/zig/tokenizer.zig
@@ -3,8 +3,12 @@ const mem = std.mem;
pub const Token = struct {
id: Id,
- start: usize,
- end: usize,
+ loc: Loc,
+
+ pub const Loc = struct {
+ start: usize,
+ end: usize,
+ };
pub const Keyword = struct {
bytes: []const u8,
@@ -426,8 +430,10 @@ pub const Tokenizer = struct {
var state: State = .start;
var result = Token{
.id = .Eof,
- .start = self.index,
- .end = undefined,
+ .loc = .{
+ .start = self.index,
+ .end = undefined,
+ },
};
var seen_escape_digits: usize = undefined;
var remaining_code_units: usize = undefined;
@@ -436,7 +442,7 @@ pub const Tokenizer = struct {
switch (state) {
.start => switch (c) {
' ', '\n', '\t', '\r' => {
- result.start = self.index + 1;
+ result.loc.start = self.index + 1;
},
'"' => {
state = .string_literal;
@@ -686,7 +692,7 @@ pub const Tokenizer = struct {
.identifier => switch (c) {
'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
else => {
- if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
+ if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
result.id = id;
}
break;
@@ -1313,7 +1319,7 @@ pub const Tokenizer = struct {
=> {},
.identifier => {
- if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
+ if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| {
result.id = id;
}
},
@@ -1420,7 +1426,7 @@ pub const Tokenizer = struct {
}
}
- result.end = self.index;
+ result.loc.end = self.index;
return result;
}
@@ -1430,8 +1436,10 @@ pub const Tokenizer = struct {
if (invalid_length == 0) return;
self.pending_invalid_token = .{
.id = .Invalid,
- .start = self.index,
- .end = self.index + invalid_length,
+ .loc = .{
+ .start = self.index,
+ .end = self.index + invalid_length,
+ },
};
}
src-self-hosted/translate_c.zig
@@ -247,27 +247,6 @@ pub const Context = struct {
}
};
- /// Helper function to append items to a singly linked list.
- fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher {
- assert(list.first == null);
- return .{
- .c = c,
- .it = &list.first,
- };
- }
-
- fn llpush(
- c: *Context,
- comptime T: type,
- it: *?*std.SinglyLinkedList(T).Node,
- data: T,
- ) !*?*std.SinglyLinkedList(T).Node {
- const llnode = try c.arena.create(std.SinglyLinkedList(T).Node);
- llnode.* = .{ .data = data };
- it.* = llnode;
- return &llnode.next;
- }
-
fn getMangle(c: *Context) u32 {
c.mangle_count += 1;
return c.mangle_count;