Commit f750618846
Changed files (1)
lib
std
lib/std/zig/tokenizer.zig
@@ -568,12 +568,14 @@ pub const Tokenizer = struct {
result.tag = .identifier;
state = .string_literal;
},
- else => {
- // reinterpret as a builtin
- self.index -= 1;
+ 'a'...'z', 'A'...'Z', '_' => {
state = .builtin;
result.tag = .builtin;
},
+ else => {
+ result.tag = .invalid;
+ break;
+ },
},
.ampersand => switch (c) {
@@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {
try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
}
+test "tokenizer - invalid builtin identifiers" {
+ try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
+ try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
+}
+
fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {