Commit 9de2c1b76e

Andrew Kelley <andrew@ziglang.org>
2021-05-28 22:26:16
stage1: tokenizer: backport recent stage2 changes
This backports ec10595b65490126e1b7d231f655be6d7d198acf stage2 tokenizer logic to stage1.
master
1 parent 01e0a4b
Changed files (1)
src
stage1
tokenizer.cpp
src/stage1/tokenizer.cpp
@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {
                     case DIGIT:
                         t.state = TokenizeState_float_fraction_dec;
                         break;
-                    case ALPHA_EXCEPT_E:
+                    default:
                         invalid_char_error(&t, c);
                         break;
-                    default:
-                        t.state = TokenizeState_start;
-                        continue;
                 }
                 break;
             case TokenizeState_num_dot_hex:
@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {
                         t.out->ids.last() = TokenIdFloatLiteral;
                         t.state = TokenizeState_float_fraction_hex;
                         break;
-                    case ALPHA_EXCEPT_HEX_AND_P:
+                    default:
                         invalid_char_error(&t, c);
                         break;
-                    default:
-                        t.state = TokenizeState_start;
-                        continue;
                 }
                 break;
             case TokenizeState_float_fraction_dec_no_underscore: