Commit 841b38aae8

Veikka Tuominen <git@vexu.eu>
2023-01-17 19:39:19
tokenizer: detect null as non-first byte of a line comment
Line comments do not produce actual tokens so they need special handling for null bytes. Closes #14346
master
1 parent fc06699
Changed files (1)
lib
std
zig
tokenizer.zig
lib/std/zig/tokenizer.zig
@@ -1151,7 +1151,13 @@ pub const Tokenizer = struct {
                     },
                 },
                 .line_comment => switch (c) {
-                    0 => break,
+                    0 => {
+                        if (self.index != self.buffer.len) {
+                            result.tag = .invalid;
+                            self.index += 1;
+                        }
+                        break;
+                    },
                     '\n' => {
                         state = .start;
                         result.loc.start = self.index + 1;
@@ -1865,6 +1871,9 @@ test "null byte before eof" {
     try testTokenize("//\x00", &.{.invalid});
     try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
     try testTokenize("\x00", &.{.invalid});
+    try testTokenize("// NUL\x00\n", &.{.invalid});
+    try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
+    try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
 }
 
 fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {