Commit `fa9f1d2396`

Andrew Kelley <andrew@ziglang.org>

2019-10-19 09:23:23

add -DZIG_ENABLE_MEM_PROFILE option and -fmem-report flag

This can be used to find what's taking up memory in the compiler. Here's an example of how to use it: ``` ./zig test ../lib/std/std.zig --cache off -fmem-report ``` And here's the output I get for this on x86_64-linux-gnu today: ``` Const: 6462833 items, 152 bytes each, total 936.84 MiB ConstGlobalRefs: 17236534 items, 24 bytes each, total 394.51 MiB ResetResult: 1698108 items, 160 bytes each, total 259.11 MiB ConstExprValue: 3118299 items, 80 bytes each, total 237.91 MiB EndExpr: 1345395 items, 168 bytes each, total 215.56 MiB Unknown_8: 27370821 items, 8 bytes each, total 208.82 MiB VarPtr: 1127866 items, 168 bytes each, total 180.70 MiB IrBasicBlock: 794834 items, 120 bytes each, total 90.96 MiB LoadPtr: 554024 items, 160 bytes each, total 84.54 MiB Unknown_64: 1245715 items, 64 bytes each, total 76.03 MiB Unknown_40: 1879218 items, 40 bytes each, total 71.69 MiB Unknown_72: 989117 items, 72 bytes each, total 67.92 MiB Return: 423783 items, 160 bytes each, total 64.66 MiB Unknown_168: 332480 items, 168 bytes each, total 53.27 MiB Unknown_152: 336890 items, 152 bytes each, total 48.84 MiB AddImplicitReturnType: 230819 items, 168 bytes each, total 36.98 MiB Br: 217835 items, 168 bytes each, total 34.90 MiB Unknown_184: 179529 items, 184 bytes each, total 31.50 MiB FieldPtr: 179388 items, 184 bytes each, total 31.48 MiB BinOp: 171004 items, 176 bytes each, total 28.70 MiB LoadPtrGen: 173287 items, 168 bytes each, total 27.76 MiB CondBr: 137864 items, 192 bytes each, total 25.24 MiB Unknown_720: 30918 items, 720 bytes each, total 21.23 MiB CallSrc: 99320 items, 216 bytes each, total 20.46 MiB Unknown_160: 129243 items, 160 bytes each, total 19.72 MiB Unknown_1: 19339456 items, 1 bytes each, total 18.44 MiB CheckStatementIsVoid: 119838 items, 160 bytes each, total 18.29 MiB Unknown_48: 371178 items, 48 bytes each, total 16.99 MiB TestComptime: 101443 items, 160 bytes each, total 15.48 MiB DeclVarSrc: 72578 items, 184 bytes each, total 12.74 MiB StorePtr: 72776 items, 176 bytes each, total 12.22 MiB ZigVar: 79201 items, 160 bytes each, total 12.09 MiB Unknown_16: 770643 items, 16 bytes each, total 11.76 MiB Phi: 60482 items, 184 bytes each, total 10.61 MiB TestErrSrc: 66177 items, 168 bytes each, total 10.60 MiB Unknown_240: 45164 items, 240 bytes each, total 10.34 MiB ElemPtr: 58232 items, 184 bytes each, total 10.22 MiB AllocaSrc: 60053 items, 176 bytes each, total 10.08 MiB CallGen: 44873 items, 224 bytes each, total 9.59 MiB SaveErrRetAddr: 63787 items, 152 bytes each, total 9.25 MiB Unknown_112: 82283 items, 112 bytes each, total 8.79 MiB AllocaGen: 51909 items, 176 bytes each, total 8.71 MiB Unknown_24: 373599 items, 24 bytes each, total 8.55 MiB ResultLocPeer: 113683 items, 72 bytes each, total 7.81 MiB DeclRef: 36343 items, 168 bytes each, total 5.82 MiB UnwrapErrPayload: 34603 items, 168 bytes each, total 5.54 MiB Ref: 33414 items, 168 bytes each, total 5.35 MiB Unknown_104: 53882 items, 104 bytes each, total 5.34 MiB DeclVarGen: 32540 items, 168 bytes each, total 5.21 MiB StructFieldPtr: 30449 items, 176 bytes each, total 5.11 MiB UnwrapErrCode: 31508 items, 168 bytes each, total 5.05 MiB Unknown_56: 90256 items, 56 bytes each, total 4.82 MiB SpillBegin: 28722 items, 168 bytes each, total 4.60 MiB SpillEnd: 28722 items, 160 bytes each, total 4.38 MiB ResultLocReturn: 64573 items, 48 bytes each, total 2.96 MiB PtrType: 14702 items, 184 bytes each, total 2.58 MiB SliceType: 15005 items, 176 bytes each, total 2.52 MiB Unknown_176: 13326 items, 176 bytes each, total 2.24 MiB RefGen: 12881 items, 168 bytes each, total 2.06 MiB UnOp: 12102 items, 176 bytes each, total 2.03 MiB SwitchBr: 9453 items, 200 bytes each, total 1.80 MiB TestErrGen: 11143 items, 160 bytes each, total 1.70 MiB Unknown_32: 52359 items, 32 bytes each, total 1.60 MiB CheckSwitchProngs: 9094 items, 184 bytes each, total 1.60 MiB TypeOf: 9259 items, 160 bytes each, total 1.41 MiB IntCast: 8772 items, 168 bytes each, total 1.41 MiB OptionalUnwrapPtr: 8755 items, 168 bytes each, total 1.40 MiB SwitchTarget: 9094 items, 160 bytes each, total 1.39 MiB Cast: 8198 items, 176 bytes each, total 1.38 MiB WidenOrShorten: 8448 items, 160 bytes each, total 1.29 MiB ErrorUnion: 7613 items, 176 bytes each, total 1.28 MiB SliceSrc: 6249 items, 192 bytes each, total 1.14 MiB ErrWrapCode: 7133 items, 168 bytes each, total 1.14 MiB TypeName: 7328 items, 160 bytes each, total 1.12 MiB ImplicitCast: 5480 items, 176 bytes each, total 941.88 KiB ResolveResult: 5638 items, 168 bytes each, total 924.98 KiB ResultLocInstruction: 22696 items, 40 bytes each, total 886.56 KiB BitCastSrc: 4947 items, 168 bytes each, total 811.62 KiB CompileErr: 5148 items, 160 bytes each, total 804.38 KiB ReturnPtr: 5305 items, 152 bytes each, total 787.46 KiB Unreachable: 5038 items, 152 bytes each, total 747.83 KiB TestNonNull: 4716 items, 160 bytes each, total 736.88 KiB BitCastGen: 4431 items, 160 bytes each, total 692.34 KiB PtrToInt: 4289 items, 160 bytes each, total 670.16 KiB SliceGen: 3573 items, 192 bytes each, total 669.94 KiB ArrayType: 4081 items, 168 bytes each, total 669.54 KiB IntType: 3868 items, 168 bytes each, total 634.59 KiB Unknown_88: 7213 items, 88 bytes each, total 619.87 KiB Truncate: 3771 items, 168 bytes each, total 618.68 KiB TypeInfo: 3740 items, 160 bytes each, total 584.38 KiB SwitchVar: 3385 items, 176 bytes each, total 581.80 KiB ContainerInitFields: 3223 items, 184 bytes each, total 579.13 KiB ContainerInitList: 2309 items, 192 bytes each, total 432.94 KiB PtrCastGen: 2626 items, 168 bytes each, total 430.83 KiB BoolNot: 2457 items, 160 bytes each, total 383.91 KiB FnProto: 2054 items, 184 bytes each, total 369.08 KiB MergeErrSets: 1927 items, 176 bytes each, total 331.20 KiB Unknown_136: 2486 items, 136 bytes each, total 330.17 KiB Unknown_80: 4059 items, 80 bytes each, total 317.11 KiB Bswap: 1670 items, 168 bytes each, total 273.98 KiB TypeId: 1680 items, 160 bytes each, total 262.50 KiB PtrCastSrc: 1371 items, 176 bytes each, total 235.64 KiB ErrName: 1193 items, 160 bytes each, total 186.41 KiB UnionTag: 1120 items, 160 bytes each, total 175.00 KiB TagName: 1050 items, 160 bytes each, total 164.06 KiB SizeOf: 942 items, 160 bytes each, total 147.19 KiB MemberName: 871 items, 168 bytes each, total 142.90 KiB Import: 881 items, 160 bytes each, total 137.66 KiB PtrOfArrayToSlice: 758 items, 168 bytes each, total 124.36 KiB UnionFieldPtr: 710 items, 176 bytes each, total 122.03 KiB EnumToInt: 778 items, 160 bytes each, total 121.56 KiB CheckRuntimeScope: 700 items, 168 bytes each, total 114.84 KiB FieldParentPtr: 632 items, 184 bytes each, total 113.56 KiB BoolToInt: 719 items, 160 bytes each, total 112.34 KiB ResultLocPeerParent: 904 items, 104 bytes each, total 91.81 KiB IntToPtr: 537 items, 168 bytes each, total 88.10 KiB AlignOf: 561 items, 160 bytes each, total 87.66 KiB AtomicRmw: 356 items, 208 bytes each, total 72.31 KiB MemberCount: 441 items, 160 bytes each, total 68.91 KiB Memset: 342 items, 176 bytes each, total 58.78 KiB PopCount: 321 items, 168 bytes each, total 52.66 KiB AlignCast: 251 items, 168 bytes each, total 41.18 KiB IrInstruction *: 5230 items, 8 bytes each, total 40.86 KiB IrBasicBlock *: 5230 items, 8 bytes each, total 40.86 KiB TagType: 261 items, 160 bytes each, total 40.78 KiB HasDecl: 234 items, 168 bytes each, total 38.39 KiB OverflowOp: 191 items, 200 bytes each, total 37.30 KiB Export: 209 items, 176 bytes each, total 35.92 KiB SetCold: 219 items, 160 bytes each, total 34.22 KiB ReturnAddress: 216 items, 152 bytes each, total 32.06 KiB FromBytes: 178 items, 176 bytes each, total 30.59 KiB SetRuntimeSafety: 188 items, 160 bytes each, total 29.38 KiB OptionalWrap: 151 items, 168 bytes each, total 24.77 KiB Clz: 143 items, 168 bytes each, total 23.46 KiB ResizeSlice: 135 items, 168 bytes each, total 22.15 KiB UnionInitNamedField: 106 items, 184 bytes each, total 19.05 KiB Panic: 102 items, 160 bytes each, total 15.94 KiB SwitchElseVar: 93 items, 168 bytes each, total 15.26 KiB ToBytes: 89 items, 168 bytes each, total 14.60 KiB IntToFloat: 78 items, 168 bytes each, total 12.80 KiB Unknown_4360: 3 items, 4360 bytes each, total 12.77 KiB ErrWrapPayload: 72 items, 168 bytes each, total 11.81 KiB FloatOp: 62 items, 176 bytes each, total 10.66 KiB FloatToInt: 47 items, 168 bytes each, total 7.71 KiB FloatCast: 46 items, 168 bytes each, total 7.55 KiB ErrToInt: 47 items, 160 bytes each, total 7.34 KiB Asm: 33 items, 216 bytes each, total 6.96 KiB ErrSetCast: 40 items, 168 bytes each, total 6.56 KiB Memcpy: 34 items, 176 bytes each, total 5.84 KiB AtomicLoad: 17 items, 184 bytes each, total 3.05 KiB AwaitSrc: 16 items, 168 bytes each, total 2.62 KiB Resume: 14 items, 160 bytes each, total 2.19 KiB AwaitGen: 12 items, 176 bytes each, total 2.06 KiB ArgType: 12 items, 168 bytes each, total 1.97 KiB AnyFrameType: 12 items, 160 bytes each, total 1.88 KiB SuspendFinish: 10 items, 160 bytes each, total 1.56 KiB SuspendBegin: 10 items, 160 bytes each, total 1.56 KiB Ctz: 9 items, 168 bytes each, total 1.48 KiB FrameHandle: 8 items, 152 bytes each, total 1.19 KiB SetEvalBranchQuota: 7 items, 160 bytes each, total 1.09 KiB AssertZero: 7 items, 160 bytes each, total 1.09 KiB UndeclaredIdent: 7 items, 160 bytes each, total 1.09 KiB CmpxchgSrc: 5 items, 216 bytes each, total 1.05 KiB CmpxchgGen: 5 items, 200 bytes each, total 1000.00 bytes IntToEnum: 4 items, 168 bytes each, total 672.00 bytes VectorType: 4 items, 168 bytes each, total 672.00 bytes ErrorReturnTrace: 2 items, 160 bytes each, total 320.00 bytes Breakpoint: 2 items, 152 bytes each, total 304.00 bytes FrameAddress: 2 items, 152 bytes each, total 304.00 bytes Unknown_4: 61 items, 4 bytes each, total 244.00 bytes VectorToArray: 1 items, 168 bytes each, total 168.00 bytes SetAlignStack: 1 items, 160 bytes each, total 160.00 bytes Unknown_12: 2 items, 12 bytes each, total 24.00 bytes ErrorTableEntry *: 0 items, 8 bytes each, total 0.00 bytes AstNode *: 0 items, 8 bytes each, total 0.00 bytes Total bytes used: 3.51 GiB ``` You can see that most of the memory is taken up by IR instructions, as well as comptime values. This points toward 2 changes which will greatly reduce memory usage: * Rework semantic analysis so that IR instructions can be freed. Currently the comptime value struct (ConstExprValue) is embedded directly into the IrInstruction struct. If this is made to be separate, at the very least pass 1 IR instructions can be freed. This includes `Const` which is the largest usage of memory currently. * Rework the ConstExprValue struct to no longer be a tagged union. For example, there's no need for an integer comptime value to be 80 bytes. From this you can also see that this eliminates some things from being the culprit. Before doing this analysis, I considered whether doing string interning would help. From the above output, you can see that all strings in the compiler account for only 18 MiB, so string interning would have been a dead end.

master

1 parent e42d86b

Changed files (12)

src

@@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_
 
 
 ConstExprValue *create_const_vals(size_t count) {
-    ConstGlobalRefs *global_refs = allocate<ConstGlobalRefs>(count);
-    ConstExprValue *vals = allocate<ConstExprValue>(count);
+    ConstGlobalRefs *global_refs = allocate<ConstGlobalRefs>(count, "ConstGlobalRefs");
+    ConstExprValue *vals = allocate<ConstExprValue>(count, "ConstExprValue");
     for (size_t i = 0; i < count; i += 1) {
         vals[i].global_refs = &global_refs[i];
     }

@@ -13,9 +13,6 @@
 #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@
 #define ZIG_VERSION_STRING "@ZIG_VERSION@"
 
-// Only used for running tests before installing.
-#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test"
-
 // Used for communicating build information to self hosted build.
 #define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@"
 #define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@"
@@ -24,4 +21,6 @@
 #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@"
 #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@"
 
+#cmakedefine ZIG_ENABLE_MEM_PROFILE
+
 #endif

@@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) {
 
 static void tree_print(FILE *f, ZigType *ty, size_t indent);
 
-static void pretty_print_bytes(FILE *f, double n) {
-    if (n > 1024.0 * 1024.0 * 1024.0) {
-        fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
-        return;
-    }
-    if (n > 1024.0 * 1024.0) {
-        fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
-        return;
-    }
-    if (n > 1024.0) {
-        fprintf(f, "%.02f KiB", n / 1024.0);
-        return;
-    }
-    fprintf(f, "%.02f bytes", n );
-    return;
-}
-
 static int compare_type_abi_sizes_desc(const void *a, const void *b) {
     uint64_t size_a = (*(ZigType * const*)(a))->abi_size;
     uint64_t size_b = (*(ZigType * const*)(b))->abi_size;
@@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) {
 
     start_peer(f, indent);
     fprintf(f, "\"sizef\": \"");
-    pretty_print_bytes(f, ty->abi_size);
+    zig_pretty_print_bytes(f, ty->abi_size);
     fprintf(f, "\"");
 
     start_peer(f, indent);

@@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) {
 }
 
 static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) {
-    IrBasicBlock *result = allocate<IrBasicBlock>(1);
+    IrBasicBlock *result = allocate<IrBasicBlock>(1, "IrBasicBlock");
     result->scope = scope;
     result->name_hint = name_hint;
     result->debug_id = exec_next_debug_id(irb->exec);
@@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) {
 
 template<typename T>
 static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) {
-    T *special_instruction = allocate<T>(1);
+    const char *name = nullptr;
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    T *dummy = nullptr;
+    name = ir_instruction_type_str(ir_instruction_id(dummy));
+#endif
+    T *special_instruction = allocate<T>(1, name);
     special_instruction->base.id = ir_instruction_id(special_instruction);
     special_instruction->base.scope = scope;
     special_instruction->base.source_node = source_node;
     special_instruction->base.debug_id = exec_next_debug_id(irb->exec);
     special_instruction->base.owner_bb = irb->current_basic_block;
-    special_instruction->base.value.global_refs = allocate<ConstGlobalRefs>(1);
+    special_instruction->base.value.global_refs = allocate<ConstGlobalRefs>(1, "ConstGlobalRefs");
     return special_instruction;
 }
 
@@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
     switch (node->data.return_expr.kind) {
         case ReturnKindUnconditional:
             {
-                ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
+                ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1, "ResultLocReturn");
                 result_loc_ret->base.id = ResultLocIdReturn;
                 ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
 
@@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
                 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr));
                 IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val,
                         SpillIdRetErrCode);
-                ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
+                ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1, "ResultLocReturn");
                 result_loc_ret->base.id = ResultLocIdReturn;
                 ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
                 ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base);
@@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s
         Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime,
         bool skip_name_check)
 {
-    ZigVar *variable_entry = allocate<ZigVar>(1);
+    ZigVar *variable_entry = allocate<ZigVar>(1, "ZigVar");
     variable_entry->parent_scope = parent_scope;
     variable_entry->shadowable = is_shadowable;
     variable_entry->mem_slot_index = SIZE_MAX;
@@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n
 }
 
 static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) {
-    ResultLocPeer *result = allocate<ResultLocPeer>(1);
+    ResultLocPeer *result = allocate<ResultLocPeer>(1, "ResultLocPeer");
     result->base.id = ResultLocIdPeer;
     result->base.source_instruction = peer_parent->base.source_instruction;
     result->parent = peer_parent;
@@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode
         scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node,
                 ir_should_inline(irb->exec, parent_scope));
 
-        scope_block->peer_parent = allocate<ResultLocPeerParent>(1);
+        scope_block->peer_parent = allocate<ResultLocPeerParent>(1, "ResultLocPeerParent");
         scope_block->peer_parent->base.id = ResultLocIdPeerParent;
         scope_block->peer_parent->base.source_instruction = scope_block->is_comptime;
         scope_block->peer_parent->end_bb = scope_block->end_block;
@@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node)
     if (lvalue == irb->codegen->invalid_instruction)
         return irb->codegen->invalid_instruction;
 
-    ResultLocInstruction *result_loc_inst = allocate<ResultLocInstruction>(1);
+    ResultLocInstruction *result_loc_inst = allocate<ResultLocInstruction>(1, "ResultLocInstruction");
     result_loc_inst->base.id = ResultLocIdInstruction;
     result_loc_inst->base.source_instruction = lvalue;
     ir_ref_instruction(lvalue, irb->current_basic_block);
@@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node
 
     ir_set_cursor_at_end_and_append_block(irb, true_block);
 
-    IrInstruction **incoming_values = allocate<IrInstruction *>(2);
+    IrInstruction **incoming_values = allocate<IrInstruction *>(2, "IrInstruction *");
     incoming_values[0] = val1;
     incoming_values[1] = val2;
-    IrBasicBlock **incoming_blocks = allocate<IrBasicBlock *>(2);
+    IrBasicBlock **incoming_blocks = allocate<IrBasicBlock *>(2, "IrBasicBlock *");
     incoming_blocks[0] = post_val1_block;
     incoming_blocks[1] = post_val2_block;
 
@@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A
     err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size;
     err_set_type->data.error_set.errors = allocate<ErrorTableEntry *>(err_count);
 
-    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(irb->codegen->errors_by_index.length + err_count);
+    size_t errors_count = irb->codegen->errors_by_index.length + err_count;
+    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
 
     for (uint32_t i = 0; i < err_count; i += 1) {
         AstNode *field_node = node->data.err_set_decl.decls.at(i);
@@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A
         }
         errors[err->value] = err;
     }
-    free(errors);
+    deallocate(errors, errors_count, "ErrorTableEntry *");
     return ir_build_const_type(irb, parent_scope, node, err_set_type);
 }
 
@@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp
     if (type_is_global_error_set(set2)) {
         return set1;
     }
-    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(ira->codegen->errors_by_index.length);
+    size_t errors_count = ira->codegen->errors_by_index.length;
+    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
     populate_error_set_table(errors, set1);
     ZigList<ErrorTableEntry *> intersection_list = {};
 
@@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp
             buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name));
         }
     }
-    free(errors);
+    deallocate(errors, errors_count, "ErrorTableEntry *");
 
     err_set_type->data.error_set.err_count = intersection_list.length;
     err_set_type->data.error_set.errors = intersection_list.items;
@@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted
             return result;
         }
 
-        ErrorTableEntry **errors = allocate<ErrorTableEntry *>(g->errors_by_index.length);
+        size_t errors_count = g->errors_by_index.length;
+        ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
         for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) {
             ErrorTableEntry *error_entry = container_set->data.error_set.errors[i];
             assert(errors[error_entry->value] == nullptr);
@@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted
                 result.data.error_set_mismatch->missing_errors.append(contained_error_entry);
             }
         }
-        free(errors);
+        deallocate(errors, errors_count, "ErrorTableEntry *");
         return result;
     }
 
@@ -14686,14 +14694,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira,
         return ira->codegen->invalid_instruction;
     }
 
-    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(ira->codegen->errors_by_index.length);
+    size_t errors_count = ira->codegen->errors_by_index.length;
+    ErrorTableEntry **errors = allocate<ErrorTableEntry *>(errors_count, "ErrorTableEntry *");
     for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) {
         ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i];
         assert(errors[error_entry->value] == nullptr);
         errors[error_entry->value] = error_entry;
     }
     ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name);
-    free(errors);
+    deallocate(errors, errors_count, "ErrorTableEntry *");
 
     return ir_const_type(ira, &instruction->base, result_type);
 }
@@ -24034,7 +24043,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira,
             return ira->codegen->invalid_instruction;
         }
 
-        AstNode **field_prev_uses = allocate<AstNode *>(ira->codegen->errors_by_index.length);
+        size_t field_prev_uses_count = ira->codegen->errors_by_index.length;
+        AstNode **field_prev_uses = allocate<AstNode *>(field_prev_uses_count, "AstNode *");
 
         for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) {
             IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i];
@@ -24091,7 +24101,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira,
             }
         }
 
-        free(field_prev_uses);
+        deallocate(field_prev_uses, field_prev_uses_count, "AstNode *");
     } else if (switch_type->id == ZigTypeIdInt) {
         RangeSet rs = {0};
         for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) {

@@ -38,8 +38,8 @@ struct IrPrint {
 
 static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction);
 
-static const char* ir_instruction_type_str(IrInstruction* instruction) {
-    switch (instruction->id) {
+const char* ir_instruction_type_str(IrInstructionId id) {
+    switch (id) {
         case IrInstructionIdInvalid:
             return "Invalid";
         case IrInstructionIdShuffleVector:
@@ -387,7 +387,7 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail
     const char *ref_count = ir_has_side_effects(instruction) ?
         "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count));
     fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id,
-        ir_instruction_type_str(instruction), type_name, ref_count);
+        ir_instruction_type_str(instruction->id), type_name, ref_count);
 }
 
 static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) {

@@ -15,4 +15,6 @@
 void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass);
 void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass);
 
+const char* ir_instruction_type_str(IrInstructionId id);
+
 #endif

@@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) {
         "  -fno-PIC                     disable Position Independent Code\n"
         "  -ftime-report                print timing diagnostics\n"
         "  -fstack-report               print stack size diagnostics\n"
+#ifdef ZIG_ENABLE_MEM_PROFILE
+        "  -fmem-report                 print memory usage diagnostics\n"
+#endif
         "  -fdump-analysis              write analysis.json file with type information\n"
         "  -femit-docs                  create a docs/ dir with html documentation\n"
         "  -fno-emit-bin                skip emitting machine code\n"
@@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) {
 
 extern "C" int ZigClang_main(int argc, char **argv);
 
+#ifdef ZIG_ENABLE_MEM_PROFILE
+bool mem_report = false;
+#endif
+
+int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) {
+    if (root_progress_node != nullptr) {
+        stage2_progress_end(root_progress_node);
+    }
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    if (mem_report) {
+        memprof_dump_stats(stderr);
+    }
+#endif
+    return exit_code;
+}
+
 int main(int argc, char **argv) {
     stage2_attach_segfault_handler();
 
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    memprof_init();
+#endif
+
     char *arg0 = argv[0];
     Error err;
 
@@ -670,6 +693,13 @@ int main(int argc, char **argv) {
                 timing_info = true;
             } else if (strcmp(arg, "-fstack-report") == 0) {
                 stack_report = true;
+            } else if (strcmp(arg, "-fmem-report") == 0) {
+#ifdef ZIG_ENABLE_MEM_PROFILE
+                mem_report = true;
+#else
+                fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n");
+                return print_error_usage(arg0);
+#endif
             } else if (strcmp(arg, "-fdump-analysis") == 0) {
                 enable_dump_analysis = true;
             } else if (strcmp(arg, "-femit-docs") == 0) {
@@ -1038,16 +1068,14 @@ int main(int argc, char **argv) {
         if (in_file) {
             ZigLibCInstallation libc;
             if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true)))
-                return EXIT_FAILURE;
-            stage2_progress_end(root_progress_node);
-            return EXIT_SUCCESS;
+                return main_exit(root_progress_node, EXIT_FAILURE);
+            return main_exit(root_progress_node, EXIT_SUCCESS);
         }
         ZigLibCInstallation libc;
         if ((err = zig_libc_find_native(&libc, true)))
-            return EXIT_FAILURE;
+            return main_exit(root_progress_node, EXIT_FAILURE);
         zig_libc_render(&libc, stdout);
-        stage2_progress_end(root_progress_node);
-        return EXIT_SUCCESS;
+        return main_exit(root_progress_node, EXIT_SUCCESS);
     }
     case CmdBuiltin: {
         CodeGen *g = codegen_create(main_pkg_path, nullptr, &target,
@@ -1065,10 +1093,9 @@ int main(int argc, char **argv) {
         Buf *builtin_source = codegen_generate_builtin_source(g);
         if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) {
             fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout)));
-            return EXIT_FAILURE;
+            return main_exit(root_progress_node, EXIT_FAILURE);
         }
-        stage2_progress_end(root_progress_node);
-        return EXIT_SUCCESS;
+        return main_exit(root_progress_node, EXIT_SUCCESS);
     }
     case CmdRun:
     case CmdBuild:
@@ -1142,7 +1169,7 @@ int main(int argc, char **argv) {
                 libc = allocate<ZigLibCInstallation>(1);
                 if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) {
                     fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err));
-                    return EXIT_FAILURE;
+                    return main_exit(root_progress_node, EXIT_FAILURE);
                 }
             }
             Buf *cache_dir_buf;
@@ -1219,7 +1246,7 @@ int main(int argc, char **argv) {
             codegen_set_rdynamic(g, rdynamic);
             if (mmacosx_version_min && mios_version_min) {
                 fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n");
-                return EXIT_FAILURE;
+                return main_exit(root_progress_node, EXIT_FAILURE);
             }
 
             if (mmacosx_version_min) {
@@ -1259,6 +1286,11 @@ int main(int argc, char **argv) {
                     zig_print_stack_report(g, stdout);
 
                 if (cmd == CmdRun) {
+                    stage2_progress_end(root_progress_node);
+#ifdef ZIG_ENABLE_MEM_PROFILE
+                    memprof_dump_stats(stderr);
+#endif
+
                     const char *exec_path = buf_ptr(&g->output_file_path);
                     ZigList<const char*> args = {0};
 
@@ -1282,10 +1314,9 @@ int main(int argc, char **argv) {
                         buf_replace(&g->output_file_path, '/', '\\');
 #endif
                         if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0)
-                            return EXIT_FAILURE;
+                            return main_exit(root_progress_node, EXIT_FAILURE);
                     }
-                    stage2_progress_end(root_progress_node);
-                    return EXIT_SUCCESS;
+                    return main_exit(root_progress_node, EXIT_SUCCESS);
                 } else {
                     zig_unreachable();
                 }
@@ -1293,8 +1324,7 @@ int main(int argc, char **argv) {
                 codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland);
                 if (timing_info)
                     codegen_print_timing_report(g, stderr);
-                stage2_progress_end(root_progress_node);
-                return EXIT_SUCCESS;
+                return main_exit(root_progress_node, EXIT_SUCCESS);
             } else if (cmd == CmdTest) {
                 codegen_set_emit_file_type(g, emit_file_type);
 
@@ -1314,7 +1344,7 @@ int main(int argc, char **argv) {
 
                 if (g->disable_bin_generation) {
                     fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n");
-                    return 0;
+                    return main_exit(root_progress_node, EXIT_SUCCESS);
                 }
 
                 Buf *test_exe_path_unresolved = &g->output_file_path;
@@ -1324,7 +1354,7 @@ int main(int argc, char **argv) {
                 if (emit_file_type != EmitFileTypeBinary) {
                     fprintf(stderr, "Created %s but skipping execution because it is non executable.\n",
                             buf_ptr(test_exe_path));
-                    return 0;
+                    return main_exit(root_progress_node, EXIT_SUCCESS);
                 }
 
                 for (size_t i = 0; i < test_exec_args.length; i += 1) {
@@ -1336,7 +1366,7 @@ int main(int argc, char **argv) {
                 if (!target_can_exec(&native, &target) && test_exec_args.length == 0) {
                     fprintf(stderr, "Created %s but skipping execution because it is non-native.\n",
                             buf_ptr(test_exe_path));
-                    return 0;
+                    return main_exit(root_progress_node, EXIT_SUCCESS);
                 }
 
                 Termination term;
@@ -1348,21 +1378,20 @@ int main(int argc, char **argv) {
                     fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n");
                     fprintf(stderr, "%s\n", buf_ptr(test_exe_path));
                 }
-                stage2_progress_end(root_progress_node);
-                return (term.how == TerminationIdClean) ? term.code : -1;
+                return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1);
             } else {
                 zig_unreachable();
             }
         }
     case CmdVersion:
         printf("%s\n", ZIG_VERSION_STRING);
-        return EXIT_SUCCESS;
+        return main_exit(root_progress_node, EXIT_SUCCESS);
     case CmdZen: {
         const char *ptr;
         size_t len;
         stage2_zen(&ptr, &len);
         fwrite(ptr, len, 1, stdout);
-        return EXIT_SUCCESS;
+        return main_exit(root_progress_node, EXIT_SUCCESS);
     }
     case CmdTargets:
         return print_target_list(stdout);

@@ -0,0 +1,139 @@
+#include "memory_profiling.hpp"
+#include "hash_map.hpp"
+#include "list.hpp"
+#include "util.hpp"
+#include <string.h>
+
+#ifdef ZIG_ENABLE_MEM_PROFILE
+
+static bool str_eql_str(const char *a, const char *b) {
+    return strcmp(a, b) == 0;
+}
+
+static uint32_t str_hash(const char *s) {
+    // FNV 32-bit hash
+    uint32_t h = 2166136261;
+    for (; *s; s += 1) {
+        h = h ^ *s;
+        h = h * 16777619;
+    }
+    return h;
+}
+
+struct CountAndSize {
+    size_t item_count;
+    size_t type_size;
+};
+
+ZigList<const char *> unknown_names = {};
+HashMap<const char *, CountAndSize, str_hash, str_eql_str> usage_table = {};
+bool table_active = false;
+
+
+static const char *get_default_name(const char *name_or_null, size_t type_size) {
+    if (name_or_null != nullptr) return name_or_null;
+    if (type_size >= unknown_names.length) {
+        table_active = false;
+        unknown_names.resize(type_size + 1);
+        table_active = true;
+    }
+    if (unknown_names.at(type_size) == nullptr) {
+        char buf[100];
+        sprintf(buf, "Unknown_%zu%c", type_size, 0);
+        unknown_names.at(type_size) = strdup(buf);
+    }
+    return unknown_names.at(type_size);
+}
+
+void memprof_alloc(const char *name, size_t count, size_t type_size) {
+    if (!table_active) return;
+    if (count == 0) return;
+    // temporarily disable during table put
+    table_active = false;
+    name = get_default_name(name, type_size);
+    auto existing_entry = usage_table.put_unique(name, {count, type_size});
+    if (existing_entry != nullptr) {
+        assert(existing_entry->value.type_size == type_size); // allocated name does not match type
+        existing_entry->value.item_count += count;
+    }
+    table_active = true;
+}
+
+void memprof_dealloc(const char *name, size_t count, size_t type_size) {
+    if (!table_active) return;
+    if (count == 0) return;
+    name = get_default_name(name, type_size);
+    auto existing_entry = usage_table.maybe_get(name);
+    if (existing_entry == nullptr) {
+        zig_panic("deallocated more than allocated; compromised memory usage stats");
+    }
+    if (existing_entry->value.type_size != type_size) {
+        zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size);
+    }
+    existing_entry->value.item_count -= count;
+}
+
+void memprof_init(void) {
+    usage_table.init(1024);
+    table_active = true;
+}
+
+struct MemItem {
+    const char *type_name;
+    CountAndSize count_and_size;
+};
+
+static size_t get_bytes(const MemItem *item) {
+    return item->count_and_size.item_count * item->count_and_size.type_size;
+}
+
+static int compare_bytes_desc(const void *a, const void *b) {
+    size_t size_a = get_bytes((const MemItem *)(a));
+    size_t size_b = get_bytes((const MemItem *)(b));
+    if (size_a > size_b)
+        return -1;
+    if (size_a < size_b)
+        return 1;
+    return 0;
+}
+
+void memprof_dump_stats(FILE *file) {
+    assert(table_active);
+    // disable modifications from this function
+    table_active = false;
+
+    ZigList<MemItem> list = {};
+
+    auto it = usage_table.entry_iterator();
+    for (;;) {
+        auto *entry = it.next();
+        if (!entry)
+            break;
+
+        list.append({entry->key, entry->value});
+    }
+
+    qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc);
+
+    size_t total_bytes_used = 0;
+
+    for (size_t i = 0; i < list.length; i += 1) {
+        const MemItem *item = &list.at(i);
+        fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name,
+                item->count_and_size.item_count, item->count_and_size.type_size);
+        size_t bytes = get_bytes(item);
+        zig_pretty_print_bytes(file, bytes);
+        fprintf(file, "\n");
+
+        total_bytes_used += bytes;
+    }
+
+    fprintf(stderr, "Total bytes used: ");
+    zig_pretty_print_bytes(file, total_bytes_used);
+    fprintf(file, "\n");
+
+    list.deinit();
+    table_active = true;
+}
+
+#endif

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2019 Andrew Kelley
+ *
+ * This file is part of zig, which is MIT licensed.
+ * See http://opensource.org/licenses/MIT
+ */
+
+#ifndef ZIG_MEMORY_PROFILING_HPP
+#define ZIG_MEMORY_PROFILING_HPP
+
+#include "config.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+void memprof_init(void);
+
+void memprof_alloc(const char *name, size_t item_count, size_t type_size);
+void memprof_dealloc(const char *name, size_t item_count, size_t type_size);
+
+void memprof_dump_stats(FILE *file);
+#endif

@@ -119,3 +119,21 @@ Slice<uint8_t> SplitIterator_rest(SplitIterator *self) {
 SplitIterator memSplit(Slice<uint8_t> buffer, Slice<uint8_t> split_bytes) {
     return SplitIterator{0, buffer, split_bytes};
 }
+
+void zig_pretty_print_bytes(FILE *f, double n) {
+    if (n > 1024.0 * 1024.0 * 1024.0) {
+        fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0);
+        return;
+    }
+    if (n > 1024.0 * 1024.0) {
+        fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0);
+        return;
+    }
+    if (n > 1024.0) {
+        fprintf(f, "%.02f KiB", n / 1024.0);
+        return;
+    }
+    fprintf(f, "%.02f bytes", n );
+    return;
+}
+

@@ -8,6 +8,8 @@
 #ifndef ZIG_UTIL_HPP
 #define ZIG_UTIL_HPP
 
+#include "memory_profiling.hpp"
+
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
@@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) {
 
 
 template<typename T>
-ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
+ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) {
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    memprof_alloc(name, count, sizeof(T));
+#endif
 #ifndef NDEBUG
     // make behavior when size == 0 portable
     if (count == 0)
@@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) {
 }
 
 template<typename T>
-ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) {
+ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) {
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    memprof_alloc(name, count, sizeof(T));
+#endif
 #ifndef NDEBUG
     // make behavior when size == 0 portable
     if (count == 0)
@@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) {
 }
 
 template<typename T>
-static inline T *reallocate(T *old, size_t old_count, size_t new_count) {
+static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) {
     T *ptr = reallocate_nonzero(old, old_count, new_count);
     if (new_count > old_count) {
         memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T));
@@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) {
 }
 
 template<typename T>
-static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) {
+static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) {
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    memprof_dealloc(name, old_count, sizeof(T));
+    memprof_alloc(name, new_count, sizeof(T));
+#endif
 #ifndef NDEBUG
     // make behavior when size == 0 portable
     if (new_count == 0 && old == nullptr)
@@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count)
     return ptr;
 }
 
+template<typename T>
+static inline void deallocate(T *old, size_t count, const char *name = nullptr) {
+#ifdef ZIG_ENABLE_MEM_PROFILE
+    memprof_dealloc(name, count, sizeof(T));
+#endif
+    free(old);
+}
+
+template<typename T>
+static inline void destroy(T *old, const char *name = nullptr) {
+    return deallocate(old, 1);
+}
+
 template <typename T, size_t n>
 constexpr size_t array_length(const T (&)[n]) {
     return n;
@@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) {
     return z;
 }
 
+void zig_pretty_print_bytes(FILE *f, double n);
+
 template<typename T>
 struct Optional {
     T value;

@@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}")
 set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)")
 set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries")
 set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix")
+set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation")
 
 if(ZIG_STATIC)
     set(ZIG_STATIC_LLVM "on")
@@ -455,6 +456,7 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/ir_print.cpp"
     "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp"
     "${CMAKE_SOURCE_DIR}/src/link.cpp"
+    "${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp"
     "${CMAKE_SOURCE_DIR}/src/os.cpp"
     "${CMAKE_SOURCE_DIR}/src/parser.cpp"
     "${CMAKE_SOURCE_DIR}/src/range_set.cpp"

Commit fa9f1d2396

Commit `fa9f1d2396`