Commit 6039554b26

@@ -425,7 +425,16 @@ pub const Tokenizer = struct {
             const c = self.buffer[self.index];
             switch (state) {
                 .start => switch (c) {
-                    0 => break,
+                    0 => {
+                        if (self.index != self.buffer.len) {
+                            result.tag = .invalid;
+                            result.loc.start = self.index;
+                            self.index += 1;
+                            result.loc.end = self.index;
+                            return result;
+                        }
+                        break;
+                    },
                     ' ', '\n', '\t', '\r' => {
                         result.loc.start = self.index + 1;
                     },
@@ -1851,6 +1860,13 @@ test "saturating operators" {
     try testTokenize("-|=", &.{.minus_pipe_equal});
 }
 
+test "null byte before eof" {
+    try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
+    try testTokenize("//\x00", &.{.invalid});
+    try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
+    try testTokenize("\x00", &.{.invalid});
+}
+
 fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_token_tags) |expected_token_tag| {