Commit f750618846

Matthew Borkowski <matthew.h.borkowski@gmail.com>
2021-05-26 05:44:55
stop tokenizer from recognizing lone `@` or `@` followed by a digit as a builtin
1 parent 44de884
Changed files (1)
lib
lib/std/zig/tokenizer.zig
@@ -568,12 +568,14 @@ pub const Tokenizer = struct {
                         result.tag = .identifier;
                         state = .string_literal;
                     },
-                    else => {
-                        // reinterpret as a builtin
-                        self.index -= 1;
+                    'a'...'z', 'A'...'Z', '_' => {
                         state = .builtin;
                         result.tag = .builtin;
                     },
+                    else => {
+                        result.tag = .invalid;
+                        break;
+                    },
                 },
 
                 .ampersand => switch (c) {
@@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {
     try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
 }
 
+test "tokenizer - invalid builtin identifiers" {
+    try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
+    try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
+}
+
 fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {