Commit 0676c04681

Igor Stojković <stojkovic.igor@gmail.com>
2024-09-23 22:06:30
tokenizer: fix 0 byte following invalid (#21482)
closes #21481
master
1 parent 943176b
Changed files (1)
lib
std
zig
tokenizer.zig
lib/std/zig/tokenizer.zig
@@ -535,6 +535,8 @@ pub const Tokenizer = struct {
                 switch (self.buffer[self.index]) {
                     0 => if (self.index == self.buffer.len) {
                         result.tag = .invalid;
+                    } else {
+                        continue :state .invalid;
                     },
                     '\n' => result.tag = .invalid,
                     else => continue :state .invalid,
@@ -1250,6 +1252,7 @@ test "invalid token characters" {
 
 test "invalid literal/comment characters" {
     try testTokenize("\"\x00\"", &.{.invalid});
+    try testTokenize("`\x00`", &.{.invalid});
     try testTokenize("//\x00", &.{.invalid});
     try testTokenize("//\x1f", &.{.invalid});
     try testTokenize("//\x7f", &.{.invalid});