Commit 9de452f9a6

Daniele Cocca <daniele.cocca@gmail.com>
2021-07-01 12:54:09
Skip over CRs at the end of multiline literals
Fixes #9257. This is needed when tokenizing input containing DOS line endings, i.e. the CRLF sequence.
master
1 parent b936bbd
Changed files (1)
lib
std
zig
tokenizer.zig
lib/std/zig/tokenizer.zig
@@ -845,7 +845,7 @@ pub const Tokenizer = struct {
                         self.index += 1;
                         break;
                     },
-                    '\t' => {},
+                    '\t', '\r' => {},
                     else => self.checkLiteralCharacter(),
                 },
 
@@ -1936,6 +1936,10 @@ test "tokenizer - invalid builtin identifiers" {
     try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
 }
 
+test "tokenizer - multiline string literal with a DOS-encoded CRLF line endings (issue #9257)" {
+    try testTokenize("\\\\foobar\r\n", &.{.multiline_string_literal_line});
+}
+
 fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {