Commit 116a99d3ce
Changed files (1)
lib
docs
lib/docs/ziglexer.js
@@ -0,0 +1,2109 @@
+const Tag = {
+ invalid: "invalid",
+ identifier: "identifier",
+ string_literal: "string_literal",
+ multiline_string_literal_line: "multiline_string_literal_line",
+ char_literal: "char_literal",
+ eof: "eof",
+ builtin: "builtin",
+ number_literal: "number_literal",
+ doc_comment: "doc_comment",
+ container_doc_comment: "container_doc_comment",
+ invalid_periodasterisks: "invalid_periodasterisks",
+ bang: "bang",
+ pipe: "pipe",
+ pipe_pipe: "pipe_pipe",
+ pipe_equal: "pipe_equal",
+ equal: "equal",
+ equal_equal: "equal_equal",
+ equal_angle_bracket_right: "equal_angle_bracket_right",
+ bang_equal: "bang_equal",
+ l_paren: "l_paren",
+ r_paren: "r_paren",
+ semicolon: "semicolon",
+ percent: "percent",
+ percent_equal: "percent_equal",
+ l_brace: "l_brace",
+ r_brace: "r_brace",
+ l_bracket: "l_bracket",
+ r_bracket: "r_bracket",
+ period: "period",
+ period_asterisk: "period_asterisk",
+ ellipsis2: "ellipsis2",
+ ellipsis3: "ellipsis3",
+ caret: "caret",
+ caret_equal: "caret_equal",
+ plus: "plus",
+ plus_plus: "plus_plus",
+ plus_equal: "plus_equal",
+ plus_percent: "plus_percent",
+ plus_percent_equal: "plus_percent_equal",
+ plus_pipe: "plus_pipe",
+ plus_pipe_equal: "plus_pipe_equal",
+ minus: "minus",
+ minus_equal: "minus_equal",
+ minus_percent: "minus_percent",
+ minus_percent_equal: "minus_percent_equal",
+ minus_pipe: "minus_pipe",
+ minus_pipe_equal: "minus_pipe_equal",
+ asterisk: "asterisk",
+ asterisk_equal: "asterisk_equal",
+ asterisk_asterisk: "asterisk_asterisk",
+ asterisk_percent: "asterisk_percent",
+ asterisk_percent_equal: "asterisk_percent_equal",
+ asterisk_pipe: "asterisk_pipe",
+ asterisk_pipe_equal: "asterisk_pipe_equal",
+ arrow: "arrow",
+ colon: "colon",
+ slash: "slash",
+ slash_equal: "slash_equal",
+ comma: "comma",
+ ampersand: "ampersand",
+ ampersand_equal: "ampersand_equal",
+ question_mark: "question_mark",
+ angle_bracket_left: "angle_bracket_left",
+ angle_bracket_left_equal: "angle_bracket_left_equal",
+ angle_bracket_angle_bracket_left: "angle_bracket_angle_bracket_left",
+ angle_bracket_angle_bracket_left_equal: "angle_bracket_angle_bracket_left_equal",
+ angle_bracket_angle_bracket_left_pipe: "angle_bracket_angle_bracket_left_pipe",
+ angle_bracket_angle_bracket_left_pipe_equal: "angle_bracket_angle_bracket_left_pipe_equal",
+ angle_bracket_right: "angle_bracket_right",
+ angle_bracket_right_equal: "angle_bracket_right_equal",
+ angle_bracket_angle_bracket_right: "angle_bracket_angle_bracket_right",
+ angle_bracket_angle_bracket_right_equal: "angle_bracket_angle_bracket_right_equal",
+ tilde: "tilde",
+ keyword_addrspace: "keyword_addrspace",
+ keyword_align: "keyword_align",
+ keyword_allowzero: "keyword_allowzero",
+ keyword_and: "keyword_and",
+ keyword_anyframe: "keyword_anyframe",
+ keyword_anytype: "keyword_anytype",
+ keyword_asm: "keyword_asm",
+ keyword_async: "keyword_async",
+ keyword_await: "keyword_await",
+ keyword_break: "keyword_break",
+ keyword_callconv: "keyword_callconv",
+ keyword_catch: "keyword_catch",
+ keyword_comptime: "keyword_comptime",
+ keyword_const: "keyword_const",
+ keyword_continue: "keyword_continue",
+ keyword_defer: "keyword_defer",
+ keyword_else: "keyword_else",
+ keyword_enum: "keyword_enum",
+ keyword_errdefer: "keyword_errdefer",
+ keyword_error: "keyword_error",
+ keyword_export: "keyword_export",
+ keyword_extern: "keyword_extern",
+ keyword_fn: "keyword_fn",
+ keyword_for: "keyword_for",
+ keyword_if: "keyword_if",
+ keyword_inline: "keyword_inline",
+ keyword_noalias: "keyword_noalias",
+ keyword_noinline: "keyword_noinline",
+ keyword_nosuspend: "keyword_nosuspend",
+ keyword_opaque: "keyword_opaque",
+ keyword_or: "keyword_or",
+ keyword_orelse: "keyword_orelse",
+ keyword_packed: "keyword_packed",
+ keyword_pub: "keyword_pub",
+ keyword_resume: "keyword_resume",
+ keyword_return: "keyword_return",
+ keyword_linksection: "keyword_linksection",
+ keyword_struct: "keyword_struct",
+ keyword_suspend: "keyword_suspend",
+ keyword_switch: "keyword_switch",
+ keyword_test: "keyword_test",
+ keyword_threadlocal: "keyword_threadlocal",
+ keyword_try: "keyword_try",
+ keyword_union: "keyword_union",
+ keyword_unreachable: "keyword_unreachable",
+ keyword_usingnamespace: "keyword_usingnamespace",
+ keyword_var: "keyword_var",
+ keyword_volatile: "keyword_volatile",
+ keyword_while: "keyword_while"
+}
+
+const State = {
+ start: 0,
+ identifier: 1,
+ builtin: 2,
+ string_literal: 3,
+ string_literal_backslash: 4,
+ multiline_string_literal_line: 5,
+ char_literal: 6,
+ char_literal_backslash: 7,
+ char_literal_hex_escape: 8,
+ char_literal_unicode_escape_saw_u: 9,
+ char_literal_unicode_escape: 10,
+ char_literal_unicode_invalid: 11,
+ char_literal_unicode: 12,
+ char_literal_end: 13,
+ backslash: 14,
+ equal: 15,
+ bang: 16,
+ pipe: 17,
+ minus: 18,
+ minus_percent: 19,
+ minus_pipe: 20,
+ asterisk: 21,
+ asterisk_percent: 22,
+ asterisk_pipe: 23,
+ slash: 24,
+ line_comment_start: 25,
+ line_comment: 26,
+ doc_comment_start: 27,
+ doc_comment: 28,
+ int: 29,
+ int_exponent: 30,
+ int_period: 31,
+ float: 32,
+ float_exponent: 33,
+ ampersand: 34,
+ caret: 35,
+ percent: 36,
+ plus: 37,
+ plus_percent: 38,
+ plus_pipe: 39,
+ angle_bracket_left: 40,
+ angle_bracket_angle_bracket_left: 41,
+ angle_bracket_angle_bracket_left_pipe: 42,
+ angle_bracket_right: 43,
+ angle_bracket_angle_bracket_right: 44,
+ period: 45,
+ period_2: 46,
+ period_asterisk: 47,
+ saw_at_sign: 48,
+}
+
+const keywords = {
+ "addrspace": Tag.keyword_addrspace,
+ "align": Tag.keyword_align,
+ "allowzero": Tag.keyword_allowzero,
+ "and": Tag.keyword_and,
+ "anyframe": Tag.keyword_anyframe,
+ "anytype": Tag.keyword_anytype,
+ "asm": Tag.keyword_asm,
+ "async": Tag.keyword_async,
+ "await": Tag.keyword_await,
+ "break": Tag.keyword_break,
+ "callconv": Tag.keyword_callconv,
+ "catch": Tag.keyword_catch,
+ "comptime": Tag.keyword_comptime,
+ "const": Tag.keyword_const,
+ "continue": Tag.keyword_continue,
+ "defer": Tag.keyword_defer,
+ "else": Tag.keyword_else,
+ "enum": Tag.keyword_enum,
+ "errdefer": Tag.keyword_errdefer,
+ "error": Tag.keyword_error,
+ "export": Tag.keyword_export,
+ "extern": Tag.keyword_extern,
+ "fn": Tag.keyword_fn,
+ "for": Tag.keyword_for,
+ "if": Tag.keyword_if,
+ "inline": Tag.keyword_inline,
+ "noalias": Tag.keyword_noalias,
+ "noinline": Tag.keyword_noinline,
+ "nosuspend": Tag.keyword_nosuspend,
+ "opaque": Tag.keyword_opaque,
+ "or": Tag.keyword_or,
+ "orelse": Tag.keyword_orelse,
+ "packed": Tag.keyword_packed,
+ "pub": Tag.keyword_pub,
+ "resume": Tag.keyword_resume,
+ "return": Tag.keyword_return,
+ "linksection": Tag.keyword_linksection,
+ "struct": Tag.keyword_struct,
+ "suspend": Tag.keyword_suspend,
+ "switch": Tag.keyword_switch,
+ "test": Tag.keyword_test,
+ "threadlocal": Tag.keyword_threadlocal,
+ "try": Tag.keyword_try,
+ "union": Tag.keyword_union,
+ "unreachable": Tag.keyword_unreachable,
+ "usingnamespace": Tag.keyword_usingnamespace,
+ "var": Tag.keyword_var,
+ "volatile": Tag.keyword_volatile,
+ "while": Tag.keyword_while,
+};
+
+function make_token(tag, start, end) {
+ return {
+ tag: tag,
+ loc: {
+ start: start,
+ end: end
+ }
+ }
+
+}
+
+function dump_tokens(tokens, raw_source) {
+
+ //TODO: this is not very fast
+ function find_tag_key(tag) {
+ for (const [key, value] of Object.entries(Tag)) {
+ if (value == tag) return key;
+ }
+ }
+
+ for (let i = 0; i < tokens.length; i++) {
+ const tok = tokens[i];
+ const z = raw_source.substring(tok.loc.start, tok.loc.end).toLowerCase();
+ console.log(`${find_tag_key(tok.tag)} "${tok.tag}" '${z}'`)
+ }
+}
+
+
+
+function tokenize_zig_source(raw_source) {
+
+ var index = 0;
+ var flag = false;
+
+ let seen_escape_digits = undefined;
+ let remaining_code_units = undefined;
+
+ const next = () => {
+ let state = State.start;
+
+ var result = {
+ tag: -1,
+ loc: {
+ start: index,
+ end: undefined,
+ },
+ };
+
+ //having a while (true) loop seems like a bad idea the loop should never
+ //take more iterations than twice the length of the source code
+ const MAX_ITERATIONS = raw_source.length * 2;
+ let iterations = 0;
+
+ while (iterations <= MAX_ITERATIONS) {
+
+ if (flag) {
+ return make_token(Tag.eof, index - 2, index - 2);
+ }
+ iterations += 1; // avoid death loops
+
+ var c = raw_source[index];
+
+ if (c === undefined) {
+ c = ' '; // push the last token
+ flag = true;
+ }
+
+ switch (state) {
+ case State.start:
+ switch (c) {
+ case 0: {
+ if (index != raw_source.length) {
+ result.tag = Tag.invalid;
+ result.loc.start = index;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ result.loc.end = index;
+ return result;
+ }
+ case ' ':
+ case '\n':
+ case '\t':
+ case '\r': {
+ result.loc.start = index + 1;
+ break;
+ }
+ case '"': {
+ state = State.string_literal;
+ result.tag = Tag.string_literal;
+ break;
+ }
+ case '\'': {
+ state = State.char_literal;
+ break;
+ }
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '_': {
+ state = State.identifier;
+ result.tag = Tag.identifier;
+ break;
+ }
+ case '@': {
+ state = State.saw_at_sign;
+ break;
+ }
+ case '=': {
+ state = State.equal;
+ break;
+ }
+ case '!': {
+ state = State.bang;
+ break;
+ }
+ case '|': {
+ state = State.pipe;
+ break;
+ }
+ case '(': {
+ result.tag = Tag.l_paren;
+ index += 1;
+ result.loc.end = index;
+
+ return result;
+
+ }
+ case ')': {
+ result.tag = Tag.r_paren;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '[': {
+ result.tag = Tag.l_bracket;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case ']': {
+ result.tag = Tag.r_bracket;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case ';': {
+ result.tag = Tag.semicolon;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case ',': {
+ result.tag = Tag.comma;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '?': {
+ result.tag = Tag.question_mark;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case ':': {
+ result.tag = Tag.colon;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '%': {
+ state = State.percent; break;
+ }
+ case '*': {
+ state = State.asterisk; break;
+ }
+ case '+': {
+ state = State.plus; break;
+ }
+ case '<': {
+ state = State.angle_bracket_left; break;
+ }
+ case '>': {
+ state = State.angle_bracket_right; break;
+ }
+ case '^': {
+ state = State.caret; break;
+ }
+ case '\\': {
+ state = State.backslash;
+ result.tag = Tag.multiline_string_literal_line; break;
+ }
+ case '{': {
+ result.tag = Tag.l_brace;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '}': {
+ result.tag = Tag.r_brace;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '~': {
+ result.tag = Tag.tilde;
+ index += 1; result.loc.end = index;
+ return result;
+
+ }
+ case '.': {
+ state = State.period; break;
+ }
+ case '-': {
+ state = State.minus; break;
+ }
+ case '/': {
+ state = State.slash; break;
+ }
+ case '&': {
+ state = State.ampersand; break;
+ }
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ {
+ state = State.int;
+ result.tag = Tag.number_literal; break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ index += 1;
+ return result;
+ }
+ }
+ break;
+ case State.saw_at_sign:
+ switch (c) {
+ case '"': {
+ result.tag = Tag.identifier;
+ state = State.string_literal; break;
+ }
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '_': {
+ state = State.builtin;
+ result.tag = Tag.builtin;
+ break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.ampersand:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.ampersand_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.ampersand; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.asterisk: switch (c) {
+ case '=': {
+ result.tag = Tag.asterisk_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ case '*': {
+ result.tag = Tag.asterisk_asterisk;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ case '%': {
+ state = State.asterisk_percent; break;
+ }
+ case '|': {
+ state = State.asterisk_pipe; break;
+ }
+ default: {
+ result.tag = Tag.asterisk;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.asterisk_percent:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.asterisk_percent_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.asterisk_percent;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.asterisk_pipe:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.asterisk_pipe_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.asterisk_pipe; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.percent:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.percent_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.percent; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.plus:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.plus_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ case '+': {
+ result.tag = Tag.plus_plus;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ case '%': {
+ state = State.plus_percent; break;
+ }
+ case '|': {
+ state = State.plus_pipe; break;
+ }
+ default: {
+ result.tag = Tag.plus; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.plus_percent:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.plus_percent_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.plus_percent; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.plus_pipe:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.plus_pipe_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.plus_pipe; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.caret:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.caret_equal;
+ index += 1; result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.caret; result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.identifier:
+ switch (c) {
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '_':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': break;
+ default: {
+ // if (Token.getKeyword(buffer[result.loc.start..index])) | tag | {
+ const z = raw_source.substring(result.loc.start, index);
+ if (z in keywords) {
+ result.tag = keywords[z];
+ }
+ result.loc.end = index;
+ return result;
+ }
+
+
+ }
+ break;
+ case State.builtin: switch (c) {
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '_':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': break;
+ default: result.loc.end = index;
+ return result;
+ }
+ break;
+ case State.backslash:
+ switch (c) {
+ case '\\': {
+ state = State.multiline_string_literal_line;
+ break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.string_literal:
+ switch (c) {
+ case '\\': {
+ state = State.string_literal_backslash; break;
+ }
+ case '"': {
+ index += 1;
+ result.loc.end = index;
+
+ return result;
+ }
+ case 0: {
+ //TODO: PORT
+ // if (index == buffer.len) {
+ // result.tag = .invalid;
+ // break;
+ // } else {
+ // checkLiteralCharacter();
+ // }
+ result.loc.end = index;
+ return result;
+ }
+ case '\n': {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ //TODO: PORT
+ //default: checkLiteralCharacter(),
+ }
+ break;
+ case State.string_literal_backslash:
+ switch (c) {
+ case 0:
+ case '\n': {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ state = State.string_literal; break;
+ }
+ }
+ break;
+ case State.char_literal: switch (c) {
+ case 0: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ case '\\': {
+ state = State.char_literal_backslash;
+ break;
+ }
+ //TODO: PORT
+ // '\'', 0x80...0xbf, 0xf8...0xff => {
+ // result.tag = .invalid;
+ // break;
+ // },
+ // 0xc0...0xdf => { // 110xxxxx
+ // remaining_code_units = 1;
+ // state = .char_literal_unicode;
+ // },
+ // 0xe0...0xef => { // 1110xxxx
+ // remaining_code_units = 2;
+ // state = .char_literal_unicode;
+ // },
+ // 0xf0...0xf7 => { // 11110xxx
+ // remaining_code_units = 3;
+ // state = .char_literal_unicode;
+ // },
+
+ // case 0x80:
+ // case 0x81:
+ // case 0x82:
+ // case 0x83:
+ // case 0x84:
+ // case 0x85:
+ // case 0x86:
+ // case 0x87:
+ // case 0x88:
+ // case 0x89:
+ // case 0x8a:
+ // case 0x8b:
+ // case 0x8c:
+ // case 0x8d:
+ // case 0x8e:
+ // case 0x8f:
+ // case 0x90:
+ // case 0x91:
+ // case 0x92:
+ // case 0x93:
+ // case 0x94:
+ // case 0x95:
+ // case 0x96:
+ // case 0x97:
+ // case 0x98:
+ // case 0x99:
+ // case 0x9a:
+ // case 0x9b:
+ // case 0x9c:
+ // case 0x9d:
+ // case 0x9e:
+ // case 0x9f:
+ // case 0xa0:
+ // case 0xa1:
+ // case 0xa2:
+ // case 0xa3:
+ // case 0xa4:
+ // case 0xa5:
+ // case 0xa6:
+ // case 0xa7:
+ // case 0xa8:
+ // case 0xa9:
+ // case 0xaa:
+ // case 0xab:
+ // case 0xac:
+ // case 0xad:
+ // case 0xae:
+ // case 0xaf:
+ // case 0xb0:
+ // case 0xb1:
+ // case 0xb2:
+ // case 0xb3:
+ // case 0xb4:
+ // case 0xb5:
+ // case 0xb6:
+ // case 0xb7:
+ // case 0xb8:
+ // case 0xb9:
+ // case 0xba:
+ // case 0xbb:
+ // case 0xbc:
+ // case 0xbd:
+ // case 0xbe:
+ // case 0xbf:
+ // case 0xf8:
+ // case 0xf9:
+ // case 0xfa:
+ // case 0xfb:
+ // case 0xfc:
+ // case 0xfd:
+ // case 0xfe:
+ // case 0xff:
+ // result.tag = .invalid;
+ // break;
+ // case 0xc0:
+ // case 0xc1:
+ // case 0xc2:
+ // case 0xc3:
+ // case 0xc4:
+ // case 0xc5:
+ // case 0xc6:
+ // case 0xc7:
+ // case 0xc8:
+ // case 0xc9:
+ // case 0xca:
+ // case 0xcb:
+ // case 0xcc:
+ // case 0xcd:
+ // case 0xce:
+ // case 0xcf:
+ // case 0xd0:
+ // case 0xd1:
+ // case 0xd2:
+ // case 0xd3:
+ // case 0xd4:
+ // case 0xd5:
+ // case 0xd6:
+ // case 0xd7:
+ // case 0xd8:
+ // case 0xd9:
+ // case 0xda:
+ // case 0xdb:
+ // case 0xdc:
+ // case 0xdd:
+ // case 0xde:
+ // case 0xdf:
+ // remaining_code_units = 1;
+ // state = .char_literal_unicode;
+ // case 0xe0:
+ // case 0xe1:
+ // case 0xe2:
+ // case 0xe3:
+ // case 0xe4:
+ // case 0xe5:
+ // case 0xe6:
+ // case 0xe7:
+ // case 0xe8:
+ // case 0xe9:
+ // case 0xea:
+ // case 0xeb:
+ // case 0xec:
+ // case 0xed:
+ // case 0xee:
+ // case 0xef:
+ // remaining_code_units = 2;
+ // state = .char_literal_unicode;
+ // case 0xf0:
+ // case 0xf1:
+ // case 0xf2:
+ // case 0xf3:
+ // case 0xf4:
+ // case 0xf5:
+ // case 0xf6:
+ // case 0xf7:
+ // remaining_code_units = 3;
+ // state = .char_literal_unicode;
+
+ case '\n': {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ state = State.char_literal_end; break;
+ }
+ }
+ break;
+ case State.char_literal_backslash:
+ switch (c) {
+ case 0:
+ case '\n': {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ case 'x': {
+ state = State.char_literal_hex_escape;
+ seen_escape_digits = 0; break;
+ }
+ case 'u': {
+ state = State.char_literal_unicode_escape_saw_u; break;
+ }
+ default: {
+ state = State.char_literal_end; break;
+ }
+ }
+ break;
+ case State.char_literal_hex_escape:
+ switch (c) {
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F': {
+ seen_escape_digits += 1;
+ if (seen_escape_digits == 2) {
+ state = State.char_literal_end;
+ } break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ esult.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.char_literal_unicode_escape_saw_u:
+ switch (c) {
+ case 0: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ case '{': {
+ state = State.char_literal_unicode_escape; break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ state = State.char_literal_unicode_invalid; break;
+ }
+ }
+ break;
+ case State.char_literal_unicode_escape:
+ switch (c) {
+ case 0: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F': break;
+ case '}': {
+ state = State.char_literal_end; // too many/few digits handled later
+ break;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ state = State.char_literal_unicode_invalid; break;
+ }
+ }
+ break;
+ case State.char_literal_unicode_invalid:
+ switch (c) {
+ // Keep consuming characters until an obvious stopping point.
+ // This consolidates e.g. `u{0ab1Q}` into a single invalid token
+ // instead of creating the tokens `u{0ab1`, `Q`, `}`
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'p':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'E':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'P':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '}':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': break;
+ default: break;
+ }
+ break;
+ case State.char_literal_end:
+ switch (c) {
+ case '\'': {
+ result.tag = Tag.char_literal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.char_literal_unicode:
+ switch (c) {
+ // 0x80...0xbf => {
+ // remaining_code_units -= 1;
+ // if (remaining_code_units == 0) {
+ // state = .char_literal_end;
+ // }
+ // },
+ default: {
+ result.tag = Tag.invalid;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.multiline_string_literal_line:
+ switch (c) {
+ case 0:
+ result.loc.end = index;
+ return result;
+ case '\n': {
+
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '\t': break;
+ //TODO: PORT
+ //default: checkLiteralCharacter(),
+
+ }
+ break;
+ case State.bang:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.bang_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.bang;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.pipe:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.pipe_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '|': {
+ result.tag = Tag.pipe_pipe;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.pipe;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.equal: switch (c) {
+ case '=': {
+ result.tag = Tag.equal_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '>': {
+ result.tag = Tag.equal_angle_bracket_right;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.equal;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.minus: switch (c) {
+ case '>': {
+ result.tag = Tag.arrow;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '=': {
+ result.tag = Tag.minus_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '%': {
+ state = State.minus_percent; break;
+ }
+ case '|': {
+ state = State.minus_pipe; break;
+ }
+ default: {
+ result.tag = Tag.minus;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.minus_percent:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.minus_percent_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.minus_percent;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.minus_pipe:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.minus_pipe_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.minus_pipe;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.angle_bracket_left:
+ switch (c) {
+ case '<': {
+ state = State.angle_bracket_angle_bracket_left; break;
+ }
+ case '=': {
+ result.tag = Tag.angle_bracket_left_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.angle_bracket_left;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.angle_bracket_angle_bracket_left:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.angle_bracket_angle_bracket_left_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ case '|': {
+ state = State.angle_bracket_angle_bracket_left_pipe;
+ }
+ default: {
+ result.tag = Tag.angle_bracket_angle_bracket_left;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.angle_bracket_angle_bracket_left_pipe:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.angle_bracket_angle_bracket_left_pipe_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.angle_bracket_angle_bracket_left_pipe;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.angle_bracket_right:
+ switch (c) {
+ case '>': {
+ state = State.angle_bracket_angle_bracket_right; break;
+ }
+ case '=': {
+ result.tag = Tag.angle_bracket_right_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.angle_bracket_right;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.angle_bracket_angle_bracket_right:
+ switch (c) {
+ case '=': {
+ result.tag = Tag.angle_bracket_angle_bracket_right_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.angle_bracket_angle_bracket_right;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.period:
+ switch (c) {
+ case '.': {
+ state = State.period_2; break;
+ }
+ case '*': {
+ state = State.period_asterisk; break;
+ }
+ default: {
+ result.tag = Tag.period;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.period_2:
+ switch (c) {
+ case '.': {
+ result.tag = Tag.ellipsis3;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.ellipsis2;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.period_asterisk:
+ switch (c) {
+ case '*': {
+ result.tag = Tag.invalid_periodasterisks;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.period_asterisk;
+ result.loc.end = index;
+ return result;
+ }
+ }
+ break;
+ case State.slash:
+ switch (c) {
+ case '/': {
+ state = State.line_comment_start;
+ break;
+ }
+ case '=': {
+ result.tag = Tag.slash_equal;
+ index += 1;
+ result.loc.end = index;
+ return result;
+ }
+ default: {
+ result.tag = Tag.slash;
+ result.loc.end = index;
+ return result;
+ }
+ } break;
+ case State.line_comment_start:
+ switch (c) {
+ case 0: {
+ if (index != raw_source.length) {
+ result.tag = Tag.invalid;
+ index += 1;
+ }
+ result.loc.end = index;
+ return result;
+ }
+ case '/': {
+ state = State.doc_comment_start; break;
+ }
+ case '!': {
+ result.tag = Tag.container_doc_comment;
+ state = State.doc_comment; break;
+ }
+ case '\n': {
+ state = State.start;
+ result.loc.start = index + 1; break;
+ }
+ case '\t':
+ state = State.line_comment; break;
+ default: {
+ state = State.line_comment;
+ //TODO: PORT
+ //checkLiteralCharacter();
+ break;
+ }
+ } break;
+ case State.doc_comment_start:
+ switch (c) {
+ case '/': {
+ state = State.line_comment; break;
+ }
+ case 0:
+ case '\n':
+ {
+ result.tag = Tag.doc_comment;
+ result.loc.end = index;
+ return result;
+ }
+ case '\t': {
+ state = State.doc_comment;
+ result.tag = Tag.doc_comment; break;
+ }
+ default: {
+ state = State.doc_comment;
+ result.tag = Tag.doc_comment;
+ //TODO: PORT
+ //checkLiteralCharacter();
+ break;
+ }
+ } break;
+ case State.line_comment:
+ switch (c) {
+ case 0: {
+ if (index != raw_source.length) {
+ result.tag = Tag.invalid;
+ index += 1;
+ }
+ result.loc.end = index;
+ return result;
+ }
+ case '\n': {
+ state = State.start;
+ result.loc.start = index + 1;
+ break;
+ }
+ case '\t': break;
+ //TODO: PORT
+ //default: checkLiteralCharacter(),
+ } break;
+ case State.doc_comment:
+ switch (c) {
+ case 0://
+ case '\n':
+ result.loc.end = index;
+ return result;
+ case '\t': break;
+ //TODOL PORT
+ // default: checkLiteralCharacter(),
+ default:
+ break;
+ } break;
+ case State.int:
+ switch (c) {
+ case '.':
+ state = State.int_period;
+ break;
+ case '_':
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ break;
+ case 'e':
+ case 'E':
+ case 'p':
+ case 'P':
+ state = State.int_exponent;
+ break;
+ default: result.loc.end = index;
+ return result;
+ } break;
+ case State.int_exponent:
+ switch (c) {
+ case '-':
+ case '+':
+ {
+ ``
+ state = State.float; break;
+ }
+ default: {
+ index -= 1;
+ state = State.int; break;
+ }
+ } break;
+ case State.int_period: switch (c) {
+ case '_':
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': {
+ state = State.float; break;
+ }
+ case 'e':
+ case 'E':
+ case 'p':
+ case 'P':
+ state = State.float_exponent; break;
+ default: {
+ index -= 1;
+ result.loc.end = index;
+ return result;
+ }
+ } break;
+ case State.float:
+ switch (c) {
+ case '_':
+ case 'a':
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'f':
+ case 'g':
+ case 'h':
+ case 'i':
+ case 'j':
+ case 'k':
+ case 'l':
+ case 'm':
+ case 'n':
+ case 'o':
+ case 'q':
+ case 'r':
+ case 's':
+ case 't':
+ case 'u':
+ case 'v':
+ case 'w':
+ case 'x':
+ case 'y':
+ case 'z':
+ case 'A':
+ case 'B':
+ case 'C':
+ case 'D':
+ case 'F':
+ case 'G':
+ case 'H':
+ case 'I':
+ case 'J':
+ case 'K':
+ case 'L':
+ case 'M':
+ case 'N':
+ case 'O':
+ case 'Q':
+ case 'R':
+ case 'S':
+ case 'T':
+ case 'U':
+ case 'V':
+ case 'W':
+ case 'X':
+ case 'Y':
+ case 'Z':
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ break;
+
+ case 'e':
+ case 'E':
+ case 'p':
+ case 'P':
+ state = State.float_exponent; break;
+ default: result.loc.end = index;
+ return result;
+ } break;
+ case State.float_exponent:
+ switch (c) {
+ case '-':
+ case '+':
+ state = State.float; break;
+ default: {
+ index -= 1;
+ state = State.float; break;
+ }
+ }
+ break;
+ }
+ index += 1;
+ }
+
+ //TODO: PORT
+ // if (result.tag == Tag.eof) {
+ // if (pending_invalid_token) | token | {
+ // pending_invalid_token = null;
+ // return token;
+ // }
+ // result.loc.start = sindex;
+ // }
+
+ result.loc.end = index;
+ return result;
+
+ }
+
+ toks = []
+
+ for (let i = 0; i < raw_source.length * 2; i++) {
+ const tok = next();
+ toks.push(tok);
+
+ if (tok.tag == Tag.eof) {
+ break;
+ }
+ }
+
+ return toks;
+}
+
+
+function generate_html_for_src(src) {
+ var toks = tokenize_zig_source(src);
+ var html = [];
+
+ html.push("<pre>");
+ let offset = 0;
+ for (let z = 0; z < toks.length; z++) {
+ const t = toks[z];
+
+ if(t.tag == Tag.eof)
+ break;
+
+ const spanStart = `<span class="zig_${t.tag}">`
+ const spanEnd = `</span>`
+
+ src = `${src.slice(0, t.loc.start + offset)}` + spanStart + `${src.slice(t.loc.start + offset)}`;
+ offset += spanStart.length;
+
+ src = `${src.slice(0, t.loc.end + offset)}` + spanEnd + `${src.slice(t.loc.end + offset)}`;
+ offset += spanEnd.length;
+ }
+
+
+ html.push(src);
+ html.push("</pre>");
+
+ return html.join("");
+
+}
+
+//const fs = require('fs');
+//const src = fs.readFileSync("../std/c.zig", 'utf8');
+//console.log(generate_html_for_src(src));
+
+
+// gist for zig_lexer_test code: https://gist.github.com/Myvar/2684ba4fb86b975274629d6f21eddc7b
+// // Just for testing not to commit in pr
+// var isNode = new Function("try {return this===global;}catch(e){return false;}");
+// if (isNode()) {
+
+
+// //const s = "const std = @import(\"std\");";
+// //const toksa = tokenize_zig_source(s);
+// //dump_tokens(toksa, s);
+// //console.log(JSON.stringify(toksa));
+
+// const fs = require('fs');
+
+// function testFile(fileName) {
+// //console.log(fileName);
+// var exec = require('child_process').execFileSync;
+// var passed = true;
+// const zig_data = exec('./zig_lexer_test', [fileName]);
+// const data = fs.readFileSync(fileName, 'utf8');
+
+// const toks = tokenize_zig_source(data);
+// const a_json = toks;
+
+// // dump_tokens(a_json, data);
+// // return;
+
+// const b_json = JSON.parse(zig_data.toString());
+
+// if (a_json.length !== b_json.length) {
+// console.log("FAILED a and be is not the same length");
+// passed = false;
+// //return;
+// }
+
+// let len = a_json.length;
+// if (len >= b_json.length) len = b_json.length;
+
+// for (let i = 0; i < len; i++) {
+// const a = a_json[i];
+// const b = b_json[i];
+
+// // console.log(a.tag + " == " + b.tag);
+
+// if (a.tag !== b.tag) {
+
+// // console.log("Around here:");
+// // console.log(
+// // data.substring(b_json[i - 2].loc.start, b_json[i - 2].loc.end),
+// // data.substring(b_json[i - 1].loc.start, b_json[i - 1].loc.end),
+// // data.substring(b_json[i].loc.start, b_json[i].loc.end),
+// // data.substring(b_json[i + 1].loc.start, b_json[i + 1].loc.end),
+// // data.substring(b_json[i + 2].loc.start, b_json[i + 2].loc.end),
+// // );
+
+// console.log("TAG: a != b");
+// console.log("js", a.tag);
+// console.log("zig", b.tag);
+// passed = false;
+// return;
+// }
+
+// if (a.tag !== Tag.eof && a.loc.start !== b.loc.start) {
+// console.log("START: a != b");
+
+// console.log("js", "\"" + data.substring(a_json[i ].loc.start, a_json[i].loc.end) + "\"");
+// console.log("zig", "\"" + data.substring(b_json[i ].loc.start, b_json[i].loc.end) + "\"");
+
+
+// passed = false;
+// return;
+// }
+
+// // if (a.tag !== Tag.eof && a.loc.end !== b.loc.end) {
+// // console.log("END: a != b");
+// // // console.log("Around here:");
+// // // console.log(
+// // // // data.substring(b_json[i - 2].loc.start, b_json[i - 2].loc.end),
+// // // // data.substring(b_json[i - 1].loc.start, b_json[i - 1].loc.end),
+// // // data.substring(b_json[i ].loc.start, b_json[i].loc.end),
+// // // // data.substring(b_json[i + 1].loc.start, b_json[i + 1].loc.end),
+// // // // data.substring(b_json[i + 2].loc.start, b_json[i + 2].loc.end),
+// // // );
+// // console.log("js", "\"" + data.substring(a_json[i ].loc.start, a_json[i].loc.end) + "\"");
+// // console.log("zig", "\"" + data.substring(b_json[i ].loc.start, b_json[i].loc.end) + "\"");
+// // passed = false;
+// // return;
+// // }
+// }
+// return passed;
+// }
+// var path = require('path');
+// function fromDir(startPath, filter) {
+// if (!fs.existsSync(startPath)) {
+// console.log("no dir ", startPath);
+// return;
+// }
+// var files = fs.readdirSync(startPath);
+// for (var i = 0; i < files.length; i++) {
+// var filename = path.join(startPath, files[i]);
+// var stat = fs.lstatSync(filename);
+// if (stat.isDirectory()) {
+// fromDir(filename, filter); //recurse
+// } else if (filename.endsWith(filter)) {
+// try {
+// console.log('-- TESTING: ', filename);
+// console.log("\t\t", testFile(filename));
+// }
+// catch {
+// }
+// };
+// };
+// };
+// fromDir('../std', '.zig');
+// //console.log(testFile("/home/myvar/code/zig/lib/std/fmt/errol.zig"));
+// //console.log(testFile("test.zig"));
+// }
\ No newline at end of file