Commit f06bce5dda
Changed files (9)
doc/langref.html.in
@@ -6450,7 +6450,7 @@ ContainerInitBody = list(StructLiteralField, ",") | list(Expression, ",")
StructLiteralField = "." Symbol "=" Expression
-PrefixOp = "!" | "-" | "~" | ("*" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
+PrefixOp = "!" | "-" | "~" | (("*" | "[*]") option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType
src/all_types.hpp
@@ -625,6 +625,7 @@ struct AstNodePrefixOpExpr {
};
struct AstNodePointerType {
+ Token *star_token;
AstNode *align_expr;
BigInt *bit_offset_start;
BigInt *bit_offset_end;
src/parser.cpp
@@ -1174,6 +1174,7 @@ static PrefixOp tok_to_prefix_op(Token *token) {
static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, Token *star_tok) {
AstNode *node = ast_create_node(pc, NodeTypePointerType, star_tok);
+ node->data.pointer_type.star_token = star_tok;
Token *token = &pc->tokens->at(*token_index);
if (token->id == TokenIdKeywordAlign) {
@@ -1211,11 +1212,11 @@ static AstNode *ast_parse_pointer_type(ParseContext *pc, size_t *token_index, To
/*
PrefixOpExpression = PrefixOp ErrorSetExpr | SuffixOpExpression
-PrefixOp = "!" | "-" | "~" | ("*" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
+PrefixOp = "!" | "-" | "~" | (("*" | "[*]") option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await"
*/
static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, size_t *token_index, bool mandatory) {
Token *token = &pc->tokens->at(*token_index);
- if (token->id == TokenIdStar) {
+ if (token->id == TokenIdStar || token->id == TokenIdBracketStarBracket) {
*token_index += 1;
return ast_parse_pointer_type(pc, token_index, token);
}
src/tokenizer.cpp
@@ -219,6 +219,8 @@ enum TokenizeState {
TokenizeStateSawAtSign,
TokenizeStateCharCode,
TokenizeStateError,
+ TokenizeStateLBracket,
+ TokenizeStateLBracketStar,
};
@@ -539,8 +541,8 @@ void tokenize(Buf *buf, Tokenization *out) {
end_token(&t);
break;
case '[':
+ t.state = TokenizeStateLBracket;
begin_token(&t, TokenIdLBracket);
- end_token(&t);
break;
case ']':
begin_token(&t, TokenIdRBracket);
@@ -852,6 +854,30 @@ void tokenize(Buf *buf, Tokenization *out) {
continue;
}
break;
+ case TokenizeStateLBracket:
+ switch (c) {
+ case '*':
+ t.state = TokenizeStateLBracketStar;
+ set_token_id(&t, t.cur_tok, TokenIdBracketStarBracket);
+ break;
+ default:
+ // reinterpret as just an lbracket
+ t.pos -= 1;
+ end_token(&t);
+ t.state = TokenizeStateStart;
+ continue;
+ }
+ break;
+ case TokenizeStateLBracketStar:
+ switch (c) {
+ case ']':
+ end_token(&t);
+ t.state = TokenizeStateStart;
+ break;
+ default:
+ invalid_char_error(&t, c);
+ }
+ break;
case TokenizeStateSawPlusPercent:
switch (c) {
case '=':
@@ -1467,12 +1493,14 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateLineString:
case TokenizeStateLineStringEnd:
case TokenizeStateSawBarBar:
+ case TokenizeStateLBracket:
end_token(&t);
break;
case TokenizeStateSawDotDot:
case TokenizeStateSawBackslash:
case TokenizeStateLineStringContinue:
case TokenizeStateLineStringContinueC:
+ case TokenizeStateLBracketStar:
tokenize_error(&t, "unexpected EOF");
break;
case TokenizeStateLineComment:
@@ -1509,6 +1537,7 @@ const char * token_name(TokenId id) {
case TokenIdBitShiftRight: return ">>";
case TokenIdBitShiftRightEq: return ">>=";
case TokenIdBitXorEq: return "^=";
+ case TokenIdBracketStarBracket: return "[*]";
case TokenIdCharLiteral: return "CharLiteral";
case TokenIdCmpEq: return "==";
case TokenIdCmpGreaterOrEq: return ">=";
src/tokenizer.hpp
@@ -28,6 +28,7 @@ enum TokenId {
TokenIdBitShiftRight,
TokenIdBitShiftRightEq,
TokenIdBitXorEq,
+ TokenIdBracketStarBracket,
TokenIdCharLiteral,
TokenIdCmpEq,
TokenIdCmpGreaterOrEq,
std/zig/parse.zig
@@ -3292,7 +3292,7 @@ fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {
Token.Id.Minus => ast.Node.PrefixOp.Op{ .Negation = void{} },
Token.Id.MinusPercent => ast.Node.PrefixOp.Op{ .NegationWrap = void{} },
Token.Id.Ampersand => ast.Node.PrefixOp.Op{ .AddressOf = void{} },
- Token.Id.Asterisk, Token.Id.AsteriskAsterisk => ast.Node.PrefixOp.Op{
+ Token.Id.Asterisk, Token.Id.AsteriskAsterisk, Token.Id.BracketStarBracket => ast.Node.PrefixOp.Op{
.PtrType = ast.Node.PrefixOp.PtrInfo{
.align_info = null,
.const_token = null,
std/zig/parser_test.zig
@@ -1,3 +1,10 @@
+test "zig fmt: pointer of unknown length" {
+ try testCanonical(
+ \\fn foo(ptr: [*]u8) void {}
+ \\
+ );
+}
+
test "zig fmt: spaces around slice operator" {
try testCanonical(
\\var a = b[c..d];
std/zig/tokenizer.zig
@@ -143,6 +143,7 @@ pub const Token = struct {
FloatLiteral,
LineComment,
DocComment,
+ BracketStarBracket,
Keyword_align,
Keyword_and,
Keyword_asm,
@@ -263,6 +264,8 @@ pub const Tokenizer = struct {
Period,
Period2,
SawAtSign,
+ LBracket,
+ LBracketStar,
};
pub fn next(self: *Tokenizer) Token {
@@ -325,9 +328,7 @@ pub const Tokenizer = struct {
break;
},
'[' => {
- result.id = Token.Id.LBracket;
- self.index += 1;
- break;
+ state = State.LBracket;
},
']' => {
result.id = Token.Id.RBracket;
@@ -429,6 +430,28 @@ pub const Tokenizer = struct {
},
},
+ State.LBracket => switch (c) {
+ '*' => {
+ state = State.LBracketStar;
+ },
+ else => {
+ result.id = Token.Id.LBracket;
+ break;
+ },
+ },
+
+ State.LBracketStar => switch (c) {
+ ']' => {
+ result.id = Token.Id.BracketStarBracket;
+ self.index += 1;
+ break;
+ },
+ else => {
+ result.id = Token.Id.Invalid;
+ break;
+ },
+ },
+
State.Ampersand => switch (c) {
'=' => {
result.id = Token.Id.AmpersandEqual;
@@ -1008,6 +1031,7 @@ pub const Tokenizer = struct {
State.CharLiteralEscape2,
State.CharLiteralEnd,
State.StringLiteralBackslash,
+ State.LBracketStar,
=> {
result.id = Token.Id.Invalid;
},
@@ -1024,6 +1048,9 @@ pub const Tokenizer = struct {
State.Slash => {
result.id = Token.Id.Slash;
},
+ State.LBracket => {
+ result.id = Token.Id.LBracket;
+ },
State.Zero => {
result.id = Token.Id.IntegerLiteral;
},
@@ -1142,6 +1169,15 @@ test "tokenizer" {
testTokenize("test", []Token.Id{Token.Id.Keyword_test});
}
+test "tokenizer - unknown length pointer" {
+ testTokenize(
+ \\[*]u8
+ , []Token.Id{
+ Token.Id.BracketStarBracket,
+ Token.Id.Identifier,
+ });
+}
+
test "tokenizer - char literal with hex escape" {
testTokenize(
\\'\x1b'
std/cstr.zig
@@ -9,13 +9,13 @@ pub const line_sep = switch (builtin.os) {
else => "\n",
};
-pub fn len(ptr: *const u8) usize {
+pub fn len(ptr: [*]const u8) usize {
var count: usize = 0;
while (ptr[count] != 0) : (count += 1) {}
return count;
}
-pub fn cmp(a: *const u8, b: *const u8) i8 {
+pub fn cmp(a: [*]const u8, b: [*]const u8) i8 {
var index: usize = 0;
while (a[index] == b[index] and a[index] != 0) : (index += 1) {}
if (a[index] > b[index]) {
@@ -27,11 +27,11 @@ pub fn cmp(a: *const u8, b: *const u8) i8 {
}
}
-pub fn toSliceConst(str: *const u8) []const u8 {
+pub fn toSliceConst(str: [*]const u8) []const u8 {
return str[0..len(str)];
}
-pub fn toSlice(str: *u8) []u8 {
+pub fn toSlice(str: [*]u8) []u8 {
return str[0..len(str)];
}