Commit 975dc5a390
Changed files (3)
std/zig/ast.zig
@@ -21,6 +21,7 @@ pub const Node = struct {
FloatLiteral,
StringLiteral,
MultilineStringLiteral,
+ CharLiteral,
UndefinedLiteral,
BuiltinCall,
Call,
@@ -42,6 +43,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
+ Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
@@ -64,6 +66,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
+ Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
@@ -86,6 +89,7 @@ pub const Node = struct {
Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
+ Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(),
Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
@@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct {
}
};
+pub const NodeCharLiteral = struct {
+ base: Node,
+ token: Token,
+
+ pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node {
+ return null;
+ }
+
+ pub fn firstToken(self: &NodeCharLiteral) Token {
+ return self.token;
+ }
+
+ pub fn lastToken(self: &NodeCharLiteral) Token {
+ return self.token;
+ }
+};
+
pub const NodeUndefinedLiteral = struct {
base: Node,
token: Token,
std/zig/parser.zig
@@ -456,6 +456,18 @@ pub const Parser = struct {
try stack.append(State.AfterOperand);
continue;
},
+ Token.Id.CharLiteral => {
+ const node = try arena.create(ast.NodeCharLiteral);
+ *node = ast.NodeCharLiteral {
+ .base = self.initNode(ast.Node.Id.CharLiteral),
+ .token = token,
+ };
+ try stack.append(State {
+ .Operand = &node.base
+ });
+ try stack.append(State.AfterOperand);
+ continue;
+ },
Token.Id.MultilineStringLiteralLine => {
const node = try arena.create(ast.NodeMultilineStringLiteral);
*node = ast.NodeMultilineStringLiteral {
@@ -1451,6 +1463,10 @@ pub const Parser = struct {
const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
},
+ ast.Node.Id.CharLiteral => {
+ const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base);
+ try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token));
+ },
ast.Node.Id.MultilineStringLiteral => {
const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
try stream.print("\n");
std/zig/tokenizer.zig
@@ -73,6 +73,7 @@ pub const Token = struct {
Identifier,
StringLiteral: StrLitKind,
MultilineStringLiteralLine: StrLitKind,
+ CharLiteral,
StringIdentifier,
Eof,
Builtin,
@@ -228,6 +229,9 @@ pub const Tokenizer = struct {
StringLiteralBackslash,
MultilineStringLiteralLine,
MultilineStringLiteralLineBackslash,
+ CharLiteral,
+ CharLiteralBackslash,
+ CharLiteralEnd,
Backslash,
Equal,
Bang,
@@ -294,6 +298,9 @@ pub const Tokenizer = struct {
state = State.StringLiteral;
result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
},
+ '\'' => {
+ state = State.CharLiteral;
+ },
'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
state = State.Identifier;
result.id = Token.Id.Identifier;
@@ -579,6 +586,35 @@ pub const Tokenizer = struct {
},
},
+ State.CharLiteral => switch (c) {
+ '\\' => {
+ state = State.CharLiteralBackslash;
+ },
+ '\'' => break, // Look for this error later.
+ else => {
+ if (c < 0x20 or c == 0x7f)
+ break; // Look for this error later.
+
+ state = State.CharLiteralEnd;
+ }
+ },
+
+ State.CharLiteralBackslash => switch (c) {
+ '\n' => break, // Look for this error later.
+ else => {
+ state = State.CharLiteralEnd;
+ },
+ },
+
+ State.CharLiteralEnd => switch (c) {
+ '\'' => {
+ result.id = Token.Id.CharLiteral;
+ self.index += 1;
+ break;
+ },
+ else => break, // Look for this error later.
+ },
+
State.MultilineStringLiteralLine => switch (c) {
'\\' => {
state = State.MultilineStringLiteralLineBackslash;
@@ -847,6 +883,7 @@ pub const Tokenizer = struct {
State.FloatExponentNumber,
State.StringLiteral, // find this error later
State.MultilineStringLiteralLine,
+ State.CharLiteralEnd,
State.Builtin => {},
State.Identifier => {
@@ -863,6 +900,8 @@ pub const Tokenizer = struct {
State.SawAtSign,
State.Backslash,
State.MultilineStringLiteralLineBackslash,
+ State.CharLiteral,
+ State.CharLiteralBackslash,
State.StringLiteralBackslash => {
result.id = Token.Id.Invalid;
},
@@ -1006,9 +1045,16 @@ test "tokenizer" {
});
}
+test "tokenizer - chars" {
+ testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
+}
+
test "tokenizer - invalid token characters" {
testTokenize("#", []Token.Id{Token.Id.Invalid});
testTokenize("`", []Token.Id{Token.Id.Invalid});
+ testTokenize("'c", []Token.Id {Token.Id.Invalid});
+ testTokenize("'", []Token.Id {Token.Id.Invalid});
+ testTokenize("''", []Token.Id {Token.Id.Invalid});
}
test "tokenizer - invalid literal/comment characters" {