Commit 138dab4524
Changed files (1)
lib
std
lib/std/zig/tokenizer.zig
@@ -387,17 +387,23 @@ pub const Tokenizer = struct {
DocComment,
ContainerDocComment,
Zero,
- IntegerLiteral,
- IntegerLiteralWithRadix,
- IntegerLiteralWithRadixHex,
- NumberDot,
+ IntegerLiteralDec,
+ IntegerLiteralDecNoUnderscore,
+ IntegerLiteralBin,
+ IntegerLiteralBinNoUnderscore,
+ IntegerLiteralOct,
+ IntegerLiteralOctNoUnderscore,
+ IntegerLiteralHex,
+ IntegerLiteralHexNoUnderscore,
+ NumberDotDec,
NumberDotHex,
- FloatFraction,
+ FloatFractionDec,
+ FloatFractionDecNoUnderscore,
FloatFractionHex,
+ FloatFractionHexNoUnderscore,
FloatExponentUnsigned,
- FloatExponentUnsignedHex,
FloatExponentNumber,
- FloatExponentNumberHex,
+ FloatExponentNumberNoUnderscore,
Ampersand,
Caret,
Percent,
@@ -550,7 +556,7 @@ pub const Tokenizer = struct {
result.id = Token.Id.IntegerLiteral;
},
'1'...'9' => {
- state = State.IntegerLiteral;
+ state = State.IntegerLiteralDec;
result.id = Token.Id.IntegerLiteral;
},
else => {
@@ -1048,55 +1054,122 @@ pub const Tokenizer = struct {
else => self.checkLiteralCharacter(),
},
State.Zero => switch (c) {
- 'b', 'o' => {
- state = State.IntegerLiteralWithRadix;
+ 'b' => {
+ state = State.IntegerLiteralBinNoUnderscore;
+ },
+ 'o' => {
+ state = State.IntegerLiteralOctNoUnderscore;
},
'x' => {
- state = State.IntegerLiteralWithRadixHex;
+ state = State.IntegerLiteralHexNoUnderscore;
},
else => {
// reinterpret as a normal number
self.index -= 1;
- state = State.IntegerLiteral;
+ state = State.IntegerLiteralDec;
},
},
- State.IntegerLiteral => switch (c) {
- '.' => {
- state = State.NumberDot;
+ State.IntegerLiteralBinNoUnderscore => switch (c) {
+ '0'...'1' => {
+ state = State.IntegerLiteralBin;
},
- 'p', 'P', 'e', 'E' => {
- state = State.FloatExponentUnsigned;
+ else => {
+ result.id = Token.Id.Invalid;
+ },
+ },
+ State.IntegerLiteralBin => switch (c) {
+ '_' => {
+ state = State.IntegerLiteralBinNoUnderscore;
+ },
+ '0'...'1' => {},
+ '2'...'9', 'a'...'z', 'A'...'Z' => {
+ result.id = Token.Id.Invalid;
},
- '0'...'9' => {},
else => break,
},
- State.IntegerLiteralWithRadix => switch (c) {
+ State.IntegerLiteralOctNoUnderscore => switch (c) {
+ '0'...'7' => {
+ state = State.IntegerLiteralOct;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
+ },
+ },
+ State.IntegerLiteralOct => switch (c) {
+ '_' => {
+ state = State.IntegerLiteralOctNoUnderscore;
+ },
+ '0'...'7' => {},
+ '8'...'9', 'a'...'z', 'A'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
+ else => break,
+ },
+ State.IntegerLiteralDecNoUnderscore => switch (c) {
+ '0'...'9' => {
+ state = State.IntegerLiteralDec;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
+ },
+ },
+ State.IntegerLiteralDec => switch (c) {
+ '_' => {
+ state = State.IntegerLiteralDecNoUnderscore;
+ },
'.' => {
- state = State.NumberDot;
+ state = State.NumberDotDec;
+ result.id = Token.Id.FloatLiteral;
+ },
+ 'e', 'E' => {
+ state = State.FloatExponentUnsigned;
+ result.id = Token.Id.FloatLiteral;
},
'0'...'9' => {},
+ 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
else => break,
},
- State.IntegerLiteralWithRadixHex => switch (c) {
+ State.IntegerLiteralHexNoUnderscore => switch (c) {
+ '0'...'9', 'a'...'f', 'A'...'F' => {
+ state = State.IntegerLiteralHex;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
+ },
+ },
+ State.IntegerLiteralHex => switch (c) {
+ '_' => {
+ state = State.IntegerLiteralHexNoUnderscore;
+ },
'.' => {
state = State.NumberDotHex;
+ result.id = Token.Id.FloatLiteral;
},
'p', 'P' => {
- state = State.FloatExponentUnsignedHex;
+ state = State.FloatExponentUnsigned;
+ result.id = Token.Id.FloatLiteral;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
+ 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
else => break,
},
- State.NumberDot => switch (c) {
+ State.NumberDotDec => switch (c) {
'.' => {
self.index -= 1;
state = State.Start;
break;
},
+ 'e', 'E' => {
+ state = State.FloatExponentUnsigned;
+ },
else => {
self.index -= 1;
result.id = Token.Id.FloatLiteral;
- state = State.FloatFraction;
+ state = State.FloatFractionDecNoUnderscore;
},
},
State.NumberDotHex => switch (c) {
@@ -1105,65 +1178,98 @@ pub const Tokenizer = struct {
state = State.Start;
break;
},
+ 'p', 'P' => {
+ state = State.FloatExponentUnsigned;
+ },
else => {
self.index -= 1;
result.id = Token.Id.FloatLiteral;
- state = State.FloatFractionHex;
+ state = State.FloatFractionHexNoUnderscore;
+ },
+ },
+ State.FloatFractionDecNoUnderscore => switch (c) {
+ '0'...'9' => {
+ state = State.FloatFractionDec;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
},
},
- State.FloatFraction => switch (c) {
+ State.FloatFractionDec => switch (c) {
+ '_' => {
+ state = State.FloatFractionDecNoUnderscore;
+ },
'e', 'E' => {
state = State.FloatExponentUnsigned;
},
'0'...'9' => {},
+ 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
else => break,
},
+ State.FloatFractionHexNoUnderscore => switch (c) {
+ '0'...'9', 'a'...'f', 'A'...'F' => {
+ state = State.FloatFractionHex;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
+ },
+ },
State.FloatFractionHex => switch (c) {
+ '_' => {
+ state = State.FloatFractionHexNoUnderscore;
+ },
'p', 'P' => {
- state = State.FloatExponentUnsignedHex;
+ state = State.FloatExponentUnsigned;
},
'0'...'9', 'a'...'f', 'A'...'F' => {},
+ 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
else => break,
},
State.FloatExponentUnsigned => switch (c) {
'+', '-' => {
- state = State.FloatExponentNumber;
+ state = State.FloatExponentNumberNoUnderscore;
},
else => {
// reinterpret as a normal exponent number
self.index -= 1;
- state = State.FloatExponentNumber;
+ state = State.FloatExponentNumberNoUnderscore;
},
},
- State.FloatExponentUnsignedHex => switch (c) {
- '+', '-' => {
- state = State.FloatExponentNumberHex;
+ State.FloatExponentNumberNoUnderscore => switch (c) {
+ '0'...'9' => {
+ state = State.FloatExponentNumber;
},
else => {
- // reinterpret as a normal exponent number
- self.index -= 1;
- state = State.FloatExponentNumberHex;
+ result.id = Token.Id.Invalid;
},
},
State.FloatExponentNumber => switch (c) {
+ '_' => {
+ state = State.FloatExponentNumberNoUnderscore;
+ },
'0'...'9' => {},
- else => break,
- },
- State.FloatExponentNumberHex => switch (c) {
- '0'...'9', 'a'...'f', 'A'...'F' => {},
+ 'a'...'z', 'A'...'Z' => {
+ result.id = Token.Id.Invalid;
+ },
else => break,
},
}
} else if (self.index == self.buffer.len) {
switch (state) {
State.Start,
- State.IntegerLiteral,
- State.IntegerLiteralWithRadix,
- State.IntegerLiteralWithRadixHex,
- State.FloatFraction,
+ State.IntegerLiteralDec,
+ State.IntegerLiteralBin,
+ State.IntegerLiteralOct,
+ State.IntegerLiteralHex,
+ State.NumberDotDec,
+ State.NumberDotHex,
+ State.FloatFractionDec,
State.FloatFractionHex,
State.FloatExponentNumber,
- State.FloatExponentNumberHex,
State.StringLiteral, // find this error later
State.MultilineStringLiteralLine,
State.Builtin,
@@ -1184,10 +1290,14 @@ pub const Tokenizer = struct {
result.id = Token.Id.ContainerDocComment;
},
- State.NumberDot,
- State.NumberDotHex,
+ State.IntegerLiteralDecNoUnderscore,
+ State.IntegerLiteralBinNoUnderscore,
+ State.IntegerLiteralOctNoUnderscore,
+ State.IntegerLiteralHexNoUnderscore,
+ State.FloatFractionDecNoUnderscore,
+ State.FloatFractionHexNoUnderscore,
+ State.FloatExponentNumberNoUnderscore,
State.FloatExponentUnsigned,
- State.FloatExponentUnsignedHex,
State.SawAtSign,
State.Backslash,
State.CharLiteral,
@@ -1585,6 +1695,217 @@ test "correctly parse pointer assignment" {
});
}
+test "tokenizer - number literals decimal" {
+ testTokenize("1", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("2", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("3", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("4", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("5", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("6", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("7", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("8", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("9", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0a", &[_]Token.Id{ .Invalid });
+ testTokenize("9b", &[_]Token.Id{ .Invalid });
+ testTokenize("1z", &[_]Token.Id{ .Invalid });
+ testTokenize("1z_1", &[_]Token.Id{ .Invalid });
+ testTokenize("9z3", &[_]Token.Id{ .Invalid });
+
+ testTokenize("0_0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0001", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("01234567890", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("012_345_6789_0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{ .IntegerLiteral });
+
+ testTokenize("00_", &[_]Token.Id{ .Invalid });
+ testTokenize("0_0_", &[_]Token.Id{ .Invalid });
+ testTokenize("0__0", &[_]Token.Id{ .Invalid });
+ testTokenize("0_0f", &[_]Token.Id{ .Invalid });
+ testTokenize("0_0_f", &[_]Token.Id{ .Invalid });
+ testTokenize("1_,", &[_]Token.Id{ .Invalid });
+
+ testTokenize("1.", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0.0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1.0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("10.0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1e0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1e100", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1.e100", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1.0e100", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1.0e+100", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1.0e-100", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{ .FloatLiteral });
+
+ testTokenize("1e", &[_]Token.Id{ .Invalid });
+ testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid });
+ testTokenize("1.0p100", &[_]Token.Id{ .Invalid });
+ testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral });
+ testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid });
+ testTokenize("1.0_,", &[_]Token.Id{ .Invalid });
+ testTokenize("1.0e,", &[_]Token.Id{ .Invalid });
+}
+
+
+test "tokenizer - number literals binary" {
+ testTokenize("0b0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b1", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b2", &[_]Token.Id{ .Invalid });
+ testTokenize("0b3", &[_]Token.Id{ .Invalid });
+ testTokenize("0b4", &[_]Token.Id{ .Invalid });
+ testTokenize("0b5", &[_]Token.Id{ .Invalid });
+ testTokenize("0b6", &[_]Token.Id{ .Invalid });
+ testTokenize("0b7", &[_]Token.Id{ .Invalid });
+ testTokenize("0b8", &[_]Token.Id{ .Invalid });
+ testTokenize("0b9", &[_]Token.Id{ .Invalid });
+ testTokenize("0ba", &[_]Token.Id{ .Invalid });
+ testTokenize("0bb", &[_]Token.Id{ .Invalid });
+ testTokenize("0bc", &[_]Token.Id{ .Invalid });
+ testTokenize("0bd", &[_]Token.Id{ .Invalid });
+ testTokenize("0be", &[_]Token.Id{ .Invalid });
+ testTokenize("0bf", &[_]Token.Id{ .Invalid });
+ testTokenize("0bz", &[_]Token.Id{ .Invalid });
+
+ testTokenize("0b0000_0000", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b1111_1111", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b10_10_10_10", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
+ testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
+
+ testTokenize("0B0", &[_]Token.Id{ .Invalid });
+ testTokenize("0b_", &[_]Token.Id{ .Invalid });
+ testTokenize("0b_0", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0b0__1", &[_]Token.Id{ .Invalid });
+ testTokenize("0b0_1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1e", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1p", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1e0", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0b1_,", &[_]Token.Id{ .Invalid });
+}
+
+test "tokenizer - number literals octal" {
+ testTokenize("0o0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o1", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o2", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o3", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o4", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o5", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o6", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o7", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o8", &[_]Token.Id{ .Invalid });
+ testTokenize("0o9", &[_]Token.Id{ .Invalid });
+ testTokenize("0oa", &[_]Token.Id{ .Invalid });
+ testTokenize("0ob", &[_]Token.Id{ .Invalid });
+ testTokenize("0oc", &[_]Token.Id{ .Invalid });
+ testTokenize("0od", &[_]Token.Id{ .Invalid });
+ testTokenize("0oe", &[_]Token.Id{ .Invalid });
+ testTokenize("0of", &[_]Token.Id{ .Invalid });
+ testTokenize("0oz", &[_]Token.Id{ .Invalid });
+
+ testTokenize("0o01234567", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o0123_4567", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o01_23_45_67", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
+ testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
+
+ testTokenize("0O0", &[_]Token.Id{ .Invalid });
+ testTokenize("0o_", &[_]Token.Id{ .Invalid });
+ testTokenize("0o_0", &[_]Token.Id{ .Invalid });
+ testTokenize("0o1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0o0__1", &[_]Token.Id{ .Invalid });
+ testTokenize("0o0_1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0o1e", &[_]Token.Id{ .Invalid });
+ testTokenize("0o1p", &[_]Token.Id{ .Invalid });
+ testTokenize("0o1e0", &[_]Token.Id{ .Invalid });
+ testTokenize("0o1p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0o_,", &[_]Token.Id{ .Invalid });
+}
+
+test "tokenizer - number literals hexadeciaml" {
+ testTokenize("0x0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x1", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x2", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x3", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x4", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x5", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x6", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x7", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x8", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x9", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xa", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xb", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xc", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xd", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xe", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xf", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xA", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xB", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xC", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xD", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xE", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0xF", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x0z", &[_]Token.Id{ .Invalid });
+ testTokenize("0xz", &[_]Token.Id{ .Invalid });
+
+ testTokenize("0x0123456789ABCDEF", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .IntegerLiteral });
+
+ testTokenize("0X0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x_", &[_]Token.Id{ .Invalid });
+ testTokenize("0x_1", &[_]Token.Id{ .Invalid });
+ testTokenize("0x1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0__1", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0_1_", &[_]Token.Id{ .Invalid });
+ testTokenize("0x_,", &[_]Token.Id{ .Invalid });
+
+ testTokenize("0x1.", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0x1.0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xF.", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xF.0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xF.F", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xF.Fp0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xF.FP0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0x1p0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xfp0", &[_]Token.Id{ .FloatLiteral });
+
+ testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0x0.0p0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xff.ffp10", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xff.ffP10", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xff.p10", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xffp10", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{ .FloatLiteral });
+ testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{ .FloatLiteral });
+
+ testTokenize("0x1e", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x1e0", &[_]Token.Id{ .IntegerLiteral });
+ testTokenize("0x1p", &[_]Token.Id{ .Invalid });
+ testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid });
+ testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0_.0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0._0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0_", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0_p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0._p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid });
+ testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid });
+}
+
+
fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {