Commit ca597e2bfb
Changed files (6)
std
src/tokenizer.cpp
@@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = {
{"false", TokenIdKeywordFalse},
{"fn", TokenIdKeywordFn},
{"for", TokenIdKeywordFor},
- {"goto", TokenIdKeywordGoto},
{"if", TokenIdKeywordIf},
{"inline", TokenIdKeywordInline},
{"nakedcc", TokenIdKeywordNakedCC},
@@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) {
case TokenIdKeywordFalse: return "false";
case TokenIdKeywordFn: return "fn";
case TokenIdKeywordFor: return "for";
- case TokenIdKeywordGoto: return "goto";
case TokenIdKeywordIf: return "if";
case TokenIdKeywordInline: return "inline";
case TokenIdKeywordNakedCC: return "nakedcc";
src/tokenizer.hpp
@@ -66,7 +66,6 @@ enum TokenId {
TokenIdKeywordFalse,
TokenIdKeywordFn,
TokenIdKeywordFor,
- TokenIdKeywordGoto,
TokenIdKeywordIf,
TokenIdKeywordInline,
TokenIdKeywordNakedCC,
std/debug/index.zig
@@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace {
pub fn dumpCurrentStackTrace() void {
const stderr = getStderrStream() catch return;
const debug_info = getSelfDebugInfo() catch |err| {
- stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;
+ stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
return;
};
defer debug_info.close();
@@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void {
pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void {
const stderr = getStderrStream() catch return;
const debug_info = getSelfDebugInfo() catch |err| {
- stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;
+ stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
return;
};
defer debug_info.close();
std/zig/ast.zig
@@ -38,11 +38,46 @@ pub const Node = struct {
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
};
}
+
+ pub fn firstToken(base: &Node) Token {
+ return switch (base.id) {
+ Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(),
+ Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(),
+ Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(),
+ Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(),
+ Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(),
+ Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(),
+ Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
+ Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
+ Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
+ Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
+ Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
+ Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
+ };
+ }
+
+ pub fn lastToken(base: &Node) Token {
+ return switch (base.id) {
+ Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(),
+ Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(),
+ Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(),
+ Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(),
+ Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(),
+ Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(),
+ Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
+ Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
+ Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
+ Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
+ Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
+ Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
+ };
+ }
};
pub const NodeRoot = struct {
base: Node,
decls: ArrayList(&Node),
+ eof_token: Token,
pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
if (index < self.decls.len) {
@@ -50,6 +85,14 @@ pub const NodeRoot = struct {
}
return null;
}
+
+ pub fn firstToken(self: &NodeRoot) Token {
+ return if (self.decls.len == 0) self.eof_token else self.decls.at(0).firstToken();
+ }
+
+ pub fn lastToken(self: &NodeRoot) Token {
+ return if (self.decls.len == 0) self.eof_token else self.decls.at(self.decls.len - 1).lastToken();
+ }
};
pub const NodeVarDecl = struct {
@@ -64,6 +107,7 @@ pub const NodeVarDecl = struct {
type_node: ?&Node,
align_node: ?&Node,
init_node: ?&Node,
+ semicolon_token: Token,
pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
var i = index;
@@ -85,6 +129,18 @@ pub const NodeVarDecl = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeVarDecl) Token {
+ if (self.visib_token) |visib_token| return visib_token;
+ if (self.comptime_token) |comptime_token| return comptime_token;
+ if (self.extern_token) |extern_token| return extern_token;
+ assert(self.lib_name == null);
+ return self.mut_token;
+ }
+
+ pub fn lastToken(self: &NodeVarDecl) Token {
+ return self.semicolon_token;
+ }
};
pub const NodeIdentifier = struct {
@@ -94,6 +150,14 @@ pub const NodeIdentifier = struct {
pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
return null;
}
+
+ pub fn firstToken(self: &NodeIdentifier) Token {
+ return self.name_token;
+ }
+
+ pub fn lastToken(self: &NodeIdentifier) Token {
+ return self.name_token;
+ }
};
pub const NodeFnProto = struct {
@@ -113,7 +177,7 @@ pub const NodeFnProto = struct {
pub const ReturnType = union(enum) {
Explicit: &Node,
- Infer,
+ Infer: Token,
InferErrorSet: &Node,
};
@@ -153,6 +217,25 @@ pub const NodeFnProto = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeFnProto) Token {
+ if (self.visib_token) |visib_token| return visib_token;
+ if (self.extern_token) |extern_token| return extern_token;
+ assert(self.lib_name == null);
+ if (self.inline_token) |inline_token| return inline_token;
+ if (self.cc_token) |cc_token| return cc_token;
+ return self.fn_token;
+ }
+
+ pub fn lastToken(self: &NodeFnProto) Token {
+ if (self.body_node) |body_node| return body_node.lastToken();
+ switch (self.return_type) {
+ // TODO allow this and next prong to share bodies since the types are the same
+ ReturnType.Explicit => |node| return node.lastToken(),
+ ReturnType.InferErrorSet => |node| return node.lastToken(),
+ ReturnType.Infer => |token| return token,
+ }
+ }
};
pub const NodeParamDecl = struct {
@@ -171,6 +254,18 @@ pub const NodeParamDecl = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeParamDecl) Token {
+ if (self.comptime_token) |comptime_token| return comptime_token;
+ if (self.noalias_token) |noalias_token| return noalias_token;
+ if (self.name_token) |name_token| return name_token;
+ return self.type_node.firstToken();
+ }
+
+ pub fn lastToken(self: &NodeParamDecl) Token {
+ if (self.var_args_token) |var_args_token| return var_args_token;
+ return self.type_node.lastToken();
+ }
};
pub const NodeBlock = struct {
@@ -187,6 +282,14 @@ pub const NodeBlock = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeBlock) Token {
+ return self.begin_token;
+ }
+
+ pub fn lastToken(self: &NodeBlock) Token {
+ return self.end_token;
+ }
};
pub const NodeInfixOp = struct {
@@ -199,6 +302,7 @@ pub const NodeInfixOp = struct {
const InfixOp = enum {
EqualEqual,
BangEqual,
+ Period,
};
pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
@@ -208,8 +312,9 @@ pub const NodeInfixOp = struct {
i -= 1;
switch (self.op) {
- InfixOp.EqualEqual => {},
- InfixOp.BangEqual => {},
+ InfixOp.EqualEqual,
+ InfixOp.BangEqual,
+ InfixOp.Period => {},
}
if (i < 1) return self.rhs;
@@ -217,6 +322,14 @@ pub const NodeInfixOp = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeInfixOp) Token {
+ return self.lhs.firstToken();
+ }
+
+ pub fn lastToken(self: &NodeInfixOp) Token {
+ return self.rhs.lastToken();
+ }
};
pub const NodePrefixOp = struct {
@@ -227,6 +340,7 @@ pub const NodePrefixOp = struct {
const PrefixOp = union(enum) {
Return,
+ Try,
AddrOf: AddrOfInfo,
};
const AddrOfInfo = struct {
@@ -241,7 +355,8 @@ pub const NodePrefixOp = struct {
var i = index;
switch (self.op) {
- PrefixOp.Return => {},
+ PrefixOp.Return,
+ PrefixOp.Try => {},
PrefixOp.AddrOf => |addr_of_info| {
if (addr_of_info.align_expr) |align_expr| {
if (i < 1) return align_expr;
@@ -255,6 +370,14 @@ pub const NodePrefixOp = struct {
return null;
}
+
+ pub fn firstToken(self: &NodePrefixOp) Token {
+ return self.op_token;
+ }
+
+ pub fn lastToken(self: &NodePrefixOp) Token {
+ return self.rhs.lastToken();
+ }
};
pub const NodeIntegerLiteral = struct {
@@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct {
pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
return null;
}
+
+ pub fn firstToken(self: &NodeIntegerLiteral) Token {
+ return self.token;
+ }
+
+ pub fn lastToken(self: &NodeIntegerLiteral) Token {
+ return self.token;
+ }
};
pub const NodeFloatLiteral = struct {
@@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct {
pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
return null;
}
+
+ pub fn firstToken(self: &NodeFloatLiteral) Token {
+ return self.token;
+ }
+
+ pub fn lastToken(self: &NodeFloatLiteral) Token {
+ return self.token;
+ }
};
pub const NodeBuiltinCall = struct {
base: Node,
builtin_token: Token,
params: ArrayList(&Node),
+ rparen_token: Token,
pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {
var i = index;
@@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct {
return null;
}
+
+ pub fn firstToken(self: &NodeBuiltinCall) Token {
+ return self.builtin_token;
+ }
+
+ pub fn lastToken(self: &NodeBuiltinCall) Token {
+ return self.rparen_token;
+ }
};
pub const NodeStringLiteral = struct {
@@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct {
pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node {
return null;
}
+
+ pub fn firstToken(self: &NodeStringLiteral) Token {
+ return self.token;
+ }
+
+ pub fn lastToken(self: &NodeStringLiteral) Token {
+ return self.token;
+ }
};
std/zig/parser.zig
@@ -69,6 +69,11 @@ pub const Parser = struct {
}
};
+ const ExpectTokenSave = struct {
+ id: Token.Id,
+ ptr: &Token,
+ };
+
const State = union(enum) {
TopLevel,
TopLevelExtern: ?Token,
@@ -85,6 +90,7 @@ pub const Parser = struct {
VarDeclAlign: &ast.NodeVarDecl,
VarDeclEq: &ast.NodeVarDecl,
ExpectToken: @TagType(Token.Id),
+ ExpectTokenSave: ExpectTokenSave,
FnProto: &ast.NodeFnProto,
FnProtoAlign: &ast.NodeFnProto,
FnProtoReturnType: &ast.NodeFnProto,
@@ -136,7 +142,10 @@ pub const Parser = struct {
stack.append(State { .TopLevelExtern = token }) catch unreachable;
continue;
},
- Token.Id.Eof => return Tree {.root_node = root_node, .arena_allocator = arena_allocator},
+ Token.Id.Eof => {
+ root_node.eof_token = token;
+ return Tree {.root_node = root_node, .arena_allocator = arena_allocator};
+ },
else => {
self.putBackToken(token);
stack.append(State { .TopLevelExtern = null }) catch unreachable;
@@ -231,13 +240,19 @@ pub const Parser = struct {
const token = self.getNextToken();
if (token.id == Token.Id.Equal) {
var_decl.eq_token = token;
- stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
+ stack.append(State {
+ .ExpectTokenSave = ExpectTokenSave {
+ .id = Token.Id.Semicolon,
+ .ptr = &var_decl.semicolon_token,
+ },
+ }) catch unreachable;
try stack.append(State {
.Expression = DestPtr {.NullableField = &var_decl.init_node},
});
continue;
}
if (token.id == Token.Id.Semicolon) {
+ var_decl.semicolon_token = token;
continue;
}
return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
@@ -247,6 +262,11 @@ pub const Parser = struct {
continue;
},
+ State.ExpectTokenSave => |expect_token_save| {
+ *expect_token_save.ptr = try self.eatToken(expect_token_save.id);
+ continue;
+ },
+
State.Expression => |dest_ptr| {
// save the dest_ptr for later
stack.append(state) catch unreachable;
@@ -264,6 +284,12 @@ pub const Parser = struct {
try stack.append(State.ExpectOperand);
continue;
},
+ Token.Id.Keyword_try => {
+ try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token,
+ ast.NodePrefixOp.PrefixOp.Try) });
+ try stack.append(State.ExpectOperand);
+ continue;
+ },
Token.Id.Ampersand => {
const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{
.AddrOf = ast.NodePrefixOp.AddrOfInfo {
@@ -306,13 +332,19 @@ pub const Parser = struct {
.base = ast.Node {.id = ast.Node.Id.BuiltinCall},
.builtin_token = token,
.params = ArrayList(&ast.Node).init(arena),
+ .rparen_token = undefined,
};
try stack.append(State {
.Operand = &node.base
});
try stack.append(State.AfterOperand);
try stack.append(State {.ExprListItemOrEnd = &node.params });
- try stack.append(State {.ExpectToken = Token.Id.LParen });
+ try stack.append(State {
+ .ExpectTokenSave = ExpectTokenSave {
+ .id = Token.Id.LParen,
+ .ptr = &node.rparen_token,
+ },
+ });
continue;
},
Token.Id.StringLiteral => {
@@ -351,6 +383,13 @@ pub const Parser = struct {
try stack.append(State.ExpectOperand);
continue;
},
+ Token.Id.Period => {
+ try stack.append(State {
+ .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period)
+ });
+ try stack.append(State.ExpectOperand);
+ continue;
+ },
else => {
// no postfix/infix operator after this operand.
self.putBackToken(token);
@@ -476,7 +515,7 @@ pub const Parser = struct {
const token = self.getNextToken();
switch (token.id) {
Token.Id.Keyword_var => {
- fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer;
+ fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token };
},
Token.Id.Bang => {
fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined };
@@ -627,6 +666,8 @@ pub const Parser = struct {
*node = ast.NodeRoot {
.base = ast.Node {.id = ast.Node.Id.Root},
.decls = ArrayList(&ast.Node).init(arena),
+ // initialized when we get the eof token
+ .eof_token = undefined,
};
return node;
}
@@ -649,6 +690,7 @@ pub const Parser = struct {
// initialized later
.name_token = undefined,
.eq_token = undefined,
+ .semicolon_token = undefined,
};
return node;
}
@@ -789,11 +831,11 @@ pub const Parser = struct {
fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
const loc = self.tokenizer.getTokenLocation(token);
- warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
+ warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);
warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
{
var i: usize = 0;
- while (i < loc.column) : (i += 1) {
+ while (i < token.column) : (i += 1) {
warn(" ");
}
}
@@ -885,11 +927,26 @@ pub const Parser = struct {
defer self.deinitUtilityArrayList(stack);
{
+ try stack.append(RenderState { .Text = "\n"});
+
var i = root_node.decls.len;
while (i != 0) {
i -= 1;
const decl = root_node.decls.items[i];
try stack.append(RenderState {.TopLevelDecl = decl});
+ if (i != 0) {
+ try stack.append(RenderState {
+ .Text = blk: {
+ const prev_node = root_node.decls.at(i - 1);
+ const prev_line_index = prev_node.lastToken().line;
+ const this_line_index = decl.firstToken().line;
+ if (this_line_index - prev_line_index >= 2) {
+ break :blk "\n\n";
+ }
+ break :blk "\n";
+ },
+ });
+ }
}
}
@@ -919,7 +976,6 @@ pub const Parser = struct {
try stream.print("(");
- try stack.append(RenderState { .Text = "\n" });
if (fn_proto.body_node == null) {
try stack.append(RenderState { .Text = ";" });
}
@@ -937,7 +993,6 @@ pub const Parser = struct {
},
ast.Node.Id.VarDecl => {
const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
- try stack.append(RenderState { .Text = "\n"});
try stack.append(RenderState { .VarDecl = var_decl});
},
@@ -1019,7 +1074,19 @@ pub const Parser = struct {
try stack.append(RenderState { .Statement = statement_node});
try stack.append(RenderState.PrintIndent);
try stack.append(RenderState { .Indent = indent + indent_delta});
- try stack.append(RenderState { .Text = "\n" });
+ try stack.append(RenderState {
+ .Text = blk: {
+ if (i != 0) {
+ const prev_statement_node = block.statements.items[i - 1];
+ const prev_line_index = prev_statement_node.lastToken().line;
+ const this_line_index = statement_node.firstToken().line;
+ if (this_line_index - prev_line_index >= 2) {
+ break :blk "\n\n";
+ }
+ }
+ break :blk "\n";
+ },
+ });
}
}
},
@@ -1033,7 +1100,9 @@ pub const Parser = struct {
ast.NodeInfixOp.InfixOp.BangEqual => {
try stack.append(RenderState { .Text = " != "});
},
- else => unreachable,
+ ast.NodeInfixOp.InfixOp.Period => {
+ try stack.append(RenderState { .Text = "."});
+ },
}
try stack.append(RenderState { .Expression = prefix_op_node.lhs });
},
@@ -1044,6 +1113,9 @@ pub const Parser = struct {
ast.NodePrefixOp.PrefixOp.Return => {
try stream.write("return ");
},
+ ast.NodePrefixOp.PrefixOp.Try => {
+ try stream.write("try ");
+ },
ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| {
try stream.write("&");
if (addr_of_info.volatile_token != null) {
@@ -1058,7 +1130,6 @@ pub const Parser = struct {
try stack.append(RenderState { .Expression = align_expr});
}
},
- else => unreachable,
}
},
ast.Node.Id.IntegerLiteral => {
@@ -1153,10 +1224,7 @@ pub const Parser = struct {
var fixed_buffer_mem: [100 * 1024]u8 = undefined;
fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
- var padded_source: [0x100]u8 = undefined;
- std.mem.copy(u8, padded_source[0..source.len], source);
-
- var tokenizer = Tokenizer.init(padded_source[0..source.len]);
+ var tokenizer = Tokenizer.init(source);
var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
defer parser.deinit();
@@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void {
}
test "zig fmt" {
+ try testCanonical(
+ \\const std = @import("std");
+ \\
+ \\pub fn main() !void {
+ \\ var stdout_file = try std.io.getStdOut;
+ \\ var stdout_file = try std.io.getStdOut;
+ \\
+ \\ var stdout_file = try std.io.getStdOut;
+ \\ var stdout_file = try std.io.getStdOut;
+ \\}
+ \\
+ );
+
try testCanonical(
\\pub fn main() !void {}
\\pub fn main() var {}
std/zig/tokenizer.zig
@@ -5,6 +5,8 @@ pub const Token = struct {
id: Id,
start: usize,
end: usize,
+ line: usize,
+ column: usize,
const KeywordId = struct {
bytes: []const u8,
@@ -16,6 +18,7 @@ pub const Token = struct {
KeywordId{.bytes="and", .id = Id.Keyword_and},
KeywordId{.bytes="asm", .id = Id.Keyword_asm},
KeywordId{.bytes="break", .id = Id.Keyword_break},
+ KeywordId{.bytes="catch", .id = Id.Keyword_catch},
KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
KeywordId{.bytes="const", .id = Id.Keyword_const},
KeywordId{.bytes="continue", .id = Id.Keyword_continue},
@@ -28,7 +31,6 @@ pub const Token = struct {
KeywordId{.bytes="false", .id = Id.Keyword_false},
KeywordId{.bytes="fn", .id = Id.Keyword_fn},
KeywordId{.bytes="for", .id = Id.Keyword_for},
- KeywordId{.bytes="goto", .id = Id.Keyword_goto},
KeywordId{.bytes="if", .id = Id.Keyword_if},
KeywordId{.bytes="inline", .id = Id.Keyword_inline},
KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
@@ -38,12 +40,14 @@ pub const Token = struct {
KeywordId{.bytes="packed", .id = Id.Keyword_packed},
KeywordId{.bytes="pub", .id = Id.Keyword_pub},
KeywordId{.bytes="return", .id = Id.Keyword_return},
+ KeywordId{.bytes="section", .id = Id.Keyword_section},
KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
KeywordId{.bytes="struct", .id = Id.Keyword_struct},
KeywordId{.bytes="switch", .id = Id.Keyword_switch},
KeywordId{.bytes="test", .id = Id.Keyword_test},
KeywordId{.bytes="this", .id = Id.Keyword_this},
KeywordId{.bytes="true", .id = Id.Keyword_true},
+ KeywordId{.bytes="try", .id = Id.Keyword_try},
KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
KeywordId{.bytes="union", .id = Id.Keyword_union},
KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
@@ -99,6 +103,7 @@ pub const Token = struct {
Keyword_and,
Keyword_asm,
Keyword_break,
+ Keyword_catch,
Keyword_comptime,
Keyword_const,
Keyword_continue,
@@ -111,7 +116,6 @@ pub const Token = struct {
Keyword_false,
Keyword_fn,
Keyword_for,
- Keyword_goto,
Keyword_if,
Keyword_inline,
Keyword_nakedcc,
@@ -121,12 +125,14 @@ pub const Token = struct {
Keyword_packed,
Keyword_pub,
Keyword_return,
+ Keyword_section,
Keyword_stdcallcc,
Keyword_struct,
Keyword_switch,
Keyword_test,
Keyword_this,
Keyword_true,
+ Keyword_try,
Keyword_undefined,
Keyword_union,
Keyword_unreachable,
@@ -140,21 +146,19 @@ pub const Token = struct {
pub const Tokenizer = struct {
buffer: []const u8,
index: usize,
+ line: usize,
+ column: usize,
pending_invalid_token: ?Token,
- pub const Location = struct {
- line: usize,
- column: usize,
+ pub const LineLocation = struct {
line_start: usize,
line_end: usize,
};
- pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
- var loc = Location {
- .line = 0,
- .column = 0,
+ pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {
+ var loc = LineLocation {
.line_start = 0,
- .line_end = 0,
+ .line_end = self.buffer.len,
};
for (self.buffer) |c, i| {
if (i == token.start) {
@@ -163,11 +167,7 @@ pub const Tokenizer = struct {
return loc;
}
if (c == '\n') {
- loc.line += 1;
- loc.column = 0;
loc.line_start = i + 1;
- } else {
- loc.column += 1;
}
}
return loc;
@@ -182,6 +182,8 @@ pub const Tokenizer = struct {
return Tokenizer {
.buffer = buffer,
.index = 0,
+ .line = 0,
+ .column = 0,
.pending_invalid_token = null,
};
}
@@ -222,13 +224,21 @@ pub const Tokenizer = struct {
.id = Token.Id.Eof,
.start = self.index,
.end = undefined,
+ .line = self.line,
+ .column = self.column,
};
- while (self.index < self.buffer.len) : (self.index += 1) {
+ while (self.index < self.buffer.len) {
const c = self.buffer[self.index];
switch (state) {
State.Start => switch (c) {
- ' ', '\n' => {
+ ' ' => {
+ result.start = self.index + 1;
+ result.column += 1;
+ },
+ '\n' => {
result.start = self.index + 1;
+ result.line += 1;
+ result.column = 0;
},
'c' => {
state = State.C;
@@ -474,6 +484,8 @@ pub const Tokenizer = struct {
result = Token {
.id = Token.Id.Eof,
.start = self.index + 1,
+ .column = 0,
+ .line = self.line + 1,
.end = undefined,
};
},
@@ -543,6 +555,14 @@ pub const Tokenizer = struct {
else => break,
},
}
+
+ self.index += 1;
+ if (c == '\n') {
+ self.line += 1;
+ self.column = 0;
+ } else {
+ self.column += 1;
+ }
} else if (self.index == self.buffer.len) {
switch (state) {
State.Start,
@@ -622,6 +642,8 @@ pub const Tokenizer = struct {
.id = Token.Id.Invalid,
.start = self.index,
.end = self.index + invalid_length,
+ .line = self.line,
+ .column = self.column,
};
}