Commit `dcbc52ec85`

Travis Staloch <twostepted@gmail.com>

2021-09-09 00:58:37

sat-arithmetic: correctly tokenize <<|, <<|=

- set state rather than result.tag in tokenizer.zig - add test to tokenizer.zig for <<, <<|, <<|=

master

1 parent b9a95f2

Changed files (3)

lib

std

zig

tokenizer.zig

src

stage1

tokenizer.cpp

Air.zig

@@ -1007,7 +1007,7 @@ pub const Tokenizer = struct {
                         break;
                     },
                     '|' => {
-                        result.tag = .angle_bracket_angle_bracket_left_pipe;
+                        state = .angle_bracket_angle_bracket_left_pipe;
                     },
                     else => {
                         result.tag = .angle_bracket_angle_bracket_left;
@@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" {
     try testTokenize("'\\u", &.{.invalid});
 }
 
+test "tokenizer - saturating" {
+    try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
+    try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
+    try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
+}
+
 fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {

@@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) {
                         t.state = TokenizeState_start;
                         break;
                     case '|':
-                        // t.out->ids.last() = TokenIdBitShiftLeftPipe;
                         t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
                         break;
                     default:

@@ -44,7 +44,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         addwrap,
-        /// Saturating integer addition. 
+        /// Saturating integer addition.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.
@@ -59,7 +59,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         subwrap,
-        /// Saturating integer subtraction. 
+        /// Saturating integer subtraction.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.
@@ -74,7 +74,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         mulwrap,
-        /// Saturating integer multiplication. 
+        /// Saturating integer multiplication.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.

Commit dcbc52ec85

Commit `dcbc52ec85`