Commit 5f5880979e
Changed files (3)
std/zig/ast.zig
@@ -6,6 +6,7 @@ const mem = std.mem;
pub const Node = struct {
id: Id,
+ comment: ?&NodeLineComment,
pub const Id = enum {
Root,
@@ -20,6 +21,7 @@ pub const Node = struct {
FloatLiteral,
StringLiteral,
BuiltinCall,
+ LineComment,
};
pub fn iterate(base: &Node, index: usize) ?&Node {
@@ -36,6 +38,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
+ Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
};
}
@@ -53,6 +56,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
+ Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
};
}
@@ -70,6 +74,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
+ Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
};
}
};
@@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct {
return self.token;
}
};
+
+pub const NodeLineComment = struct {
+ base: Node,
+ lines: ArrayList(Token),
+
+ pub fn iterate(self: &NodeLineComment, index: usize) ?&Node {
+ return null;
+ }
+
+ pub fn firstToken(self: &NodeLineComment) Token {
+ return self.lines.at(0);
+ }
+
+ pub fn lastToken(self: &NodeLineComment) Token {
+ return self.lines.at(self.lines.len - 1);
+ }
+};
std/zig/parser.zig
@@ -18,6 +18,7 @@ pub const Parser = struct {
put_back_tokens: [2]Token,
put_back_count: usize,
source_file_name: []const u8,
+ pending_line_comment_node: ?&ast.NodeLineComment,
pub const Tree = struct {
root_node: &ast.NodeRoot,
@@ -43,6 +44,7 @@ pub const Parser = struct {
.put_back_count = 0,
.source_file_name = source_file_name,
.utility_bytes = []align(utility_bytes_align) u8{},
+ .pending_line_comment_node = null,
};
}
@@ -131,6 +133,33 @@ pub const Parser = struct {
// warn("\n");
//}
+ // look for line comments
+ while (true) {
+ const token = self.getNextToken();
+ if (token.id == Token.Id.LineComment) {
+ const node = blk: {
+ if (self.pending_line_comment_node) |comment_node| {
+ break :blk comment_node;
+ } else {
+ const comment_node = try arena.create(ast.NodeLineComment);
+ *comment_node = ast.NodeLineComment {
+ .base = ast.Node {
+ .id = ast.Node.Id.LineComment,
+ .comment = null,
+ },
+ .lines = ArrayList(Token).init(arena),
+ };
+ self.pending_line_comment_node = comment_node;
+ break :blk comment_node;
+ }
+ };
+ try node.lines.append(token);
+ continue;
+ }
+ self.putBackToken(token);
+ break;
+ }
+
// This gives us 1 free append that can't fail
const state = stack.pop();
@@ -329,7 +358,7 @@ pub const Parser = struct {
Token.Id.Builtin => {
const node = try arena.create(ast.NodeBuiltinCall);
*node = ast.NodeBuiltinCall {
- .base = ast.Node {.id = ast.Node.Id.BuiltinCall},
+ .base = self.initNode(ast.Node.Id.BuiltinCall),
.builtin_token = token,
.params = ArrayList(&ast.Node).init(arena),
.rparen_token = undefined,
@@ -350,7 +379,7 @@ pub const Parser = struct {
Token.Id.StringLiteral => {
const node = try arena.create(ast.NodeStringLiteral);
*node = ast.NodeStringLiteral {
- .base = ast.Node {.id = ast.Node.Id.StringLiteral},
+ .base = self.initNode(ast.Node.Id.StringLiteral),
.token = token,
};
try stack.append(State {
@@ -359,6 +388,7 @@ pub const Parser = struct {
try stack.append(State.AfterOperand);
continue;
},
+
else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
}
},
@@ -660,11 +690,19 @@ pub const Parser = struct {
}
}
+ fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
+ if (self.pending_line_comment_node) |comment_node| {
+ self.pending_line_comment_node = null;
+ return ast.Node {.id = id, .comment = comment_node};
+ }
+ return ast.Node {.id = id, .comment = null };
+ }
+
fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot {
const node = try arena.create(ast.NodeRoot);
*node = ast.NodeRoot {
- .base = ast.Node {.id = ast.Node.Id.Root},
+ .base = self.initNode(ast.Node.Id.Root),
.decls = ArrayList(&ast.Node).init(arena),
// initialized when we get the eof token
.eof_token = undefined,
@@ -678,7 +716,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeVarDecl);
*node = ast.NodeVarDecl {
- .base = ast.Node {.id = ast.Node.Id.VarDecl},
+ .base = self.initNode(ast.Node.Id.VarDecl),
.visib_token = *visib_token,
.mut_token = *mut_token,
.comptime_token = *comptime_token,
@@ -701,7 +739,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeFnProto);
*node = ast.NodeFnProto {
- .base = ast.Node {.id = ast.Node.Id.FnProto},
+ .base = self.initNode(ast.Node.Id.FnProto),
.visib_token = *visib_token,
.name_token = null,
.fn_token = *fn_token,
@@ -722,7 +760,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeParamDecl);
*node = ast.NodeParamDecl {
- .base = ast.Node {.id = ast.Node.Id.ParamDecl},
+ .base = self.initNode(ast.Node.Id.ParamDecl),
.comptime_token = null,
.noalias_token = null,
.name_token = null,
@@ -736,7 +774,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeBlock);
*node = ast.NodeBlock {
- .base = ast.Node {.id = ast.Node.Id.Block},
+ .base = self.initNode(ast.Node.Id.Block),
.begin_token = *begin_token,
.end_token = undefined,
.statements = ArrayList(&ast.Node).init(arena),
@@ -748,7 +786,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeInfixOp);
*node = ast.NodeInfixOp {
- .base = ast.Node {.id = ast.Node.Id.InfixOp},
+ .base = self.initNode(ast.Node.Id.InfixOp),
.op_token = *op_token,
.lhs = undefined,
.op = *op,
@@ -761,7 +799,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodePrefixOp);
*node = ast.NodePrefixOp {
- .base = ast.Node {.id = ast.Node.Id.PrefixOp},
+ .base = self.initNode(ast.Node.Id.PrefixOp),
.op_token = *op_token,
.op = *op,
.rhs = undefined,
@@ -773,7 +811,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeIdentifier);
*node = ast.NodeIdentifier {
- .base = ast.Node {.id = ast.Node.Id.Identifier},
+ .base = self.initNode(ast.Node.Id.Identifier),
.name_token = *name_token,
};
return node;
@@ -783,7 +821,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeIntegerLiteral);
*node = ast.NodeIntegerLiteral {
- .base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
+ .base = self.initNode(ast.Node.Id.IntegerLiteral),
.token = *token,
};
return node;
@@ -793,7 +831,7 @@ pub const Parser = struct {
const node = try arena.create(ast.NodeFloatLiteral);
*node = ast.NodeFloatLiteral {
- .base = ast.Node {.id = ast.Node.Id.FloatLiteral},
+ .base = self.initNode(ast.Node.Id.FloatLiteral),
.token = *token,
};
return node;
@@ -1158,9 +1196,11 @@ pub const Parser = struct {
}
}
},
+ ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
+ ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
+
ast.Node.Id.Root,
ast.Node.Id.VarDecl,
- ast.Node.Id.FnProto,
ast.Node.Id.ParamDecl => unreachable,
},
RenderState.FnProtoRParen => |fn_proto| {
@@ -1187,6 +1227,12 @@ pub const Parser = struct {
}
},
RenderState.Statement => |base| {
+ if (base.comment) |comment| {
+ for (comment.lines.toSliceConst()) |line_token| {
+ try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
+ try stream.writeByteNTimes(' ', indent);
+ }
+ }
switch (base.id) {
ast.Node.Id.VarDecl => {
const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
@@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void {
}
test "zig fmt" {
+ try testCanonical(
+ \\const std = @import("std");
+ \\
+ \\pub fn main() !void {
+ \\ // If this program is run without stdout attached, exit with an error.
+ \\ // another comment
+ \\ var stdout_file = try std.io.getStdOut;
+ \\}
+ \\
+ );
+
try testCanonical(
\\const std = @import("std");
\\
std/zig/tokenizer.zig
@@ -99,6 +99,7 @@ pub const Token = struct {
AmpersandEqual,
IntegerLiteral,
FloatLiteral,
+ LineComment,
Keyword_align,
Keyword_and,
Keyword_asm,
@@ -470,7 +471,7 @@ pub const Tokenizer = struct {
State.Slash => switch (c) {
'/' => {
- result.id = undefined;
+ result.id = Token.Id.LineComment;
state = State.LineComment;
},
else => {
@@ -479,16 +480,7 @@ pub const Tokenizer = struct {
},
},
State.LineComment => switch (c) {
- '\n' => {
- state = State.Start;
- result = Token {
- .id = Token.Id.Eof,
- .start = self.index + 1,
- .column = 0,
- .line = self.line + 1,
- .end = undefined,
- };
- },
+ '\n' => break,
else => self.checkLiteralCharacter(),
},
State.Zero => switch (c) {