master
1//===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Platform-specific code.
12//===----------------------------------------------------------------------===//
13
14#ifndef TSAN_PLATFORM_H
15#define TSAN_PLATFORM_H
16
17#if !defined(__LP64__) && !defined(_WIN64)
18# error "Only 64-bit is supported"
19#endif
20
21#include "sanitizer_common/sanitizer_common.h"
22#include "tsan_defs.h"
23
24namespace __tsan {
25
26enum {
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping = 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping = 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity = 1 << 2,
37 // Meta for an app region overlaps with the meta of another app region.
38 // This is determined by recomputing the individual meta regions for
39 // each app region.
40 //
41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42 // is no MetaToMem function. However, note that (!kBrokenLinearity
43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44 kBrokenAliasedMetas = 1 << 3,
45};
46
47/*
48C/C++ on linux/x86_64 and freebsd/x86_64
490000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
500200 0000 0000 - 1000 0000 0000: -
511000 0000 0000 - 3000 0000 0000: shadow (32TB)
523000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
533800 0000 0000 - 5500 0000 0000: -
545500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
555a00 0000 0000 - 7200 0000 0000: -
567200 0000 0000 - 7300 0000 0000: heap (1TB)
577300 0000 0000 - 7a00 0000 0000: -
587a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
59
60C/C++ on netbsd/amd64 can reuse the same mapping:
61 * The address space starts from 0x1000 (option with 0x0) and ends with
62 0x7f7ffffff000.
63 * LoAppMem-kHeapMemEnd can be reused as it is.
64 * No VDSO support.
65 * No MidAppMem region.
66 * No additional HeapMem region.
67 * HiAppMem contains the stack, loader, shared libraries and heap.
68 * Stack on NetBSD/amd64 has prereserved 128MB.
69 * Heap grows downwards (top-down).
70 * ASLR must be disabled per-process or globally.
71*/
72struct Mapping48AddressSpace {
73 static const uptr kMetaShadowBeg = 0x300000000000ull;
74 static const uptr kMetaShadowEnd = 0x380000000000ull;
75 static const uptr kShadowBeg = 0x100000000000ull;
76 static const uptr kShadowEnd = 0x300000000000ull;
77 static const uptr kHeapMemBeg = 0x720000000000ull;
78 static const uptr kHeapMemEnd = 0x730000000000ull;
79 static const uptr kLoAppMemBeg = 0x000000001000ull;
80 static const uptr kLoAppMemEnd = 0x020000000000ull;
81 static const uptr kMidAppMemBeg = 0x550000000000ull;
82 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
83 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
84 static const uptr kHiAppMemEnd = 0x800000000000ull;
85 static const uptr kShadowMsk = 0x700000000000ull;
86 static const uptr kShadowXor = 0x000000000000ull;
87 static const uptr kShadowAdd = 0x100000000000ull;
88 static const uptr kVdsoBeg = 0xf000000000000000ull;
89};
90
91/*
92C/C++ on linux/mips64 (40-bit VMA)
930000 0000 00 - 0100 0000 00: - (4 GB)
940100 0000 00 - 0200 0000 00: main binary (4 GB)
950200 0000 00 - 1200 0000 00: - (64 GB)
961200 0000 00 - 2200 0000 00: shadow (64 GB)
972200 0000 00 - 4000 0000 00: - (120 GB)
984000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
995000 0000 00 - aa00 0000 00: - (360 GB)
100aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
101ab00 0000 00 - fe00 0000 00: - (332 GB)
102fe00 0000 00 - ff00 0000 00: heap (4 GB)
103ff00 0000 00 - ff80 0000 00: - (2 GB)
104ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
105*/
106struct MappingMips64_40 {
107 static const uptr kMetaShadowBeg = 0x4000000000ull;
108 static const uptr kMetaShadowEnd = 0x5000000000ull;
109 static const uptr kShadowBeg = 0x1200000000ull;
110 static const uptr kShadowEnd = 0x2200000000ull;
111 static const uptr kHeapMemBeg = 0xfe00000000ull;
112 static const uptr kHeapMemEnd = 0xff00000000ull;
113 static const uptr kLoAppMemBeg = 0x0100000000ull;
114 static const uptr kLoAppMemEnd = 0x0200000000ull;
115 static const uptr kMidAppMemBeg = 0xaa00000000ull;
116 static const uptr kMidAppMemEnd = 0xab00000000ull;
117 static const uptr kHiAppMemBeg = 0xff80000000ull;
118 static const uptr kHiAppMemEnd = 0xffffffffffull;
119 static const uptr kShadowMsk = 0xf800000000ull;
120 static const uptr kShadowXor = 0x0800000000ull;
121 static const uptr kShadowAdd = 0x0000000000ull;
122 static const uptr kVdsoBeg = 0xfffff00000ull;
123};
124
125/*
126C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
1270000 0000 00 - 0100 0000 00: - (4 GB)
1280100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
1290200 0000 00 - 0300 0000 00: heap (4 GB)
1300300 0000 00 - 0400 0000 00: - (4 GB)
1310400 0000 00 - 0800 0000 00: shadow memory (16 GB)
1320800 0000 00 - 0d00 0000 00: - (20 GB)
1330d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
1340e00 0000 00 - 1000 0000 00: -
135*/
136struct MappingAppleAarch64 {
137 static const uptr kLoAppMemBeg = 0x0100000000ull;
138 static const uptr kLoAppMemEnd = 0x0200000000ull;
139 static const uptr kHeapMemBeg = 0x0200000000ull;
140 static const uptr kHeapMemEnd = 0x0300000000ull;
141 static const uptr kShadowBeg = 0x0400000000ull;
142 static const uptr kShadowEnd = 0x0800000000ull;
143 static const uptr kMetaShadowBeg = 0x0d00000000ull;
144 static const uptr kMetaShadowEnd = 0x0e00000000ull;
145 static const uptr kHiAppMemBeg = 0x0fc0000000ull;
146 static const uptr kHiAppMemEnd = 0x0fc0000000ull;
147 static const uptr kShadowMsk = 0x0ull;
148 static const uptr kShadowXor = 0x0ull;
149 static const uptr kShadowAdd = 0x0200000000ull;
150 static const uptr kVdsoBeg = 0x7000000000000000ull;
151 static const uptr kMidAppMemBeg = 0;
152 static const uptr kMidAppMemEnd = 0;
153};
154
155/*
156C/C++ on linux/aarch64 (39-bit VMA)
1570000 0010 00 - 0500 0000 00: main binary (20 GB)
1580100 0000 00 - 2000 0000 00: -
1592000 0000 00 - 4000 0000 00: shadow memory (128 GB)
1604000 0000 00 - 4800 0000 00: metainfo (32 GB)
1614800 0000 00 - 5500 0000 00: -
1625500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
1635600 0000 00 - 7c00 0000 00: -
1647a00 0000 00 - 7d00 0000 00: heap (12 GB)
1657d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
166*/
167struct MappingAarch64_39 {
168 static const uptr kLoAppMemBeg = 0x0000001000ull;
169 static const uptr kLoAppMemEnd = 0x0500000000ull;
170 static const uptr kShadowBeg = 0x2000000000ull;
171 static const uptr kShadowEnd = 0x4000000000ull;
172 static const uptr kMetaShadowBeg = 0x4000000000ull;
173 static const uptr kMetaShadowEnd = 0x4800000000ull;
174 static const uptr kMidAppMemBeg = 0x5500000000ull;
175 static const uptr kMidAppMemEnd = 0x5a00000000ull;
176 static const uptr kHeapMemBeg = 0x7a00000000ull;
177 static const uptr kHeapMemEnd = 0x7d00000000ull;
178 static const uptr kHiAppMemBeg = 0x7d00000000ull;
179 static const uptr kHiAppMemEnd = 0x7fffffffffull;
180 static const uptr kShadowMsk = 0x7000000000ull;
181 static const uptr kShadowXor = 0x1000000000ull;
182 static const uptr kShadowAdd = 0x0000000000ull;
183 static const uptr kVdsoBeg = 0x7f00000000ull;
184};
185
186/*
187C/C++ on linux/aarch64 (42-bit VMA)
18800000 0010 00 - 02000 0000 00: main binary (128 GB)
18902000 0000 00 - 08000 0000 00: -
19010000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
19120000 0000 00 - 24000 0000 00: metainfo (256 GB)
19224000 0000 00 - 2aa00 0000 00: -
1932aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
1942c000 0000 00 - 3c000 0000 00: -
1953c000 0000 00 - 3f000 0000 00: heap (192 GB)
1963f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
197*/
198struct MappingAarch64_42 {
199 static const uptr kLoAppMemBeg = 0x00000001000ull;
200 static const uptr kLoAppMemEnd = 0x02000000000ull;
201 static const uptr kShadowBeg = 0x10000000000ull;
202 static const uptr kShadowEnd = 0x20000000000ull;
203 static const uptr kMetaShadowBeg = 0x20000000000ull;
204 static const uptr kMetaShadowEnd = 0x24000000000ull;
205 static const uptr kMidAppMemBeg = 0x2aa00000000ull;
206 static const uptr kMidAppMemEnd = 0x2c000000000ull;
207 static const uptr kHeapMemBeg = 0x3c000000000ull;
208 static const uptr kHeapMemEnd = 0x3f000000000ull;
209 static const uptr kHiAppMemBeg = 0x3f000000000ull;
210 static const uptr kHiAppMemEnd = 0x3ffffffffffull;
211 static const uptr kShadowMsk = 0x38000000000ull;
212 static const uptr kShadowXor = 0x08000000000ull;
213 static const uptr kShadowAdd = 0x00000000000ull;
214 static const uptr kVdsoBeg = 0x37f00000000ull;
215};
216
217/*
218C/C++ on linux/aarch64 (48-bit VMA)
2190000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
2200a00 0000 1000 - 1554 0000 0000: -
2211554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
2225400 0000 1000 - 8000 0000 0000: -
2238000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
224a000 0000 1000 - aaaa 0000 0000: -
225aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
226ac00 0000 1000 - fc00 0000 0000: -
227fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
228
229N.B. the shadow memory region has a strange start address, because it
230contains the shadows for the mid, high and low app regions (in this
231unusual order).
232*/
233struct MappingAarch64_48 {
234 static const uptr kLoAppMemBeg = 0x0000000001000ull;
235 static const uptr kLoAppMemEnd = 0x00a0000000000ull;
236 static const uptr kShadowBeg = 0x0155400000000ull;
237 static const uptr kShadowEnd = 0x0540000000000ull;
238 static const uptr kMetaShadowBeg = 0x0800000000000ull;
239 static const uptr kMetaShadowEnd = 0x0a00000000000ull;
240 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
241 static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
242 static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
243 static const uptr kHiAppMemEnd = 0x1000000000000ull;
244 static const uptr kHeapMemBeg = 0x0fc0000000000ull;
245 static const uptr kHeapMemEnd = 0x0fc0000000000ull;
246 static const uptr kShadowMsk = 0x0c00000000000ull;
247 static const uptr kShadowXor = 0x0200000000000ull;
248 static const uptr kShadowAdd = 0x0000000000000ull;
249 static const uptr kVdsoBeg = 0xffff000000000ull;
250};
251
252/* C/C++ on linux/loongarch64 (47-bit VMA)
2530000 0000 4000 - 0080 0000 0000: main binary
2540080 0000 0000 - 0100 0000 0000: -
2550100 0000 0000 - 1000 0000 0000: shadow memory
2561000 0000 0000 - 3000 0000 0000: -
2573000 0000 0000 - 3400 0000 0000: metainfo
2583400 0000 0000 - 5555 0000 0000: -
2595555 0000 0000 - 5556 0000 0000: main binary (PIE)
2605556 0000 0000 - 7ffe 0000 0000: -
2617ffe 0000 0000 - 7fff 0000 0000: heap
2627fff 0000 0000 - 7fff 8000 0000: -
2637fff 8000 0000 - 8000 0000 0000: modules and main thread stack
264*/
265struct MappingLoongArch64_47 {
266 static const uptr kMetaShadowBeg = 0x300000000000ull;
267 static const uptr kMetaShadowEnd = 0x340000000000ull;
268 static const uptr kShadowBeg = 0x010000000000ull;
269 static const uptr kShadowEnd = 0x100000000000ull;
270 static const uptr kHeapMemBeg = 0x7ffe00000000ull;
271 static const uptr kHeapMemEnd = 0x7fff00000000ull;
272 static const uptr kLoAppMemBeg = 0x000000004000ull;
273 static const uptr kLoAppMemEnd = 0x008000000000ull;
274 static const uptr kMidAppMemBeg = 0x555500000000ull;
275 static const uptr kMidAppMemEnd = 0x555600000000ull;
276 static const uptr kHiAppMemBeg = 0x7fff80000000ull;
277 static const uptr kHiAppMemEnd = 0x800000000000ull;
278 static const uptr kShadowMsk = 0x780000000000ull;
279 static const uptr kShadowXor = 0x040000000000ull;
280 static const uptr kShadowAdd = 0x000000000000ull;
281 static const uptr kVdsoBeg = 0x7fffffffc000ull;
282};
283
284/*
285C/C++ on linux/powerpc64 (44-bit VMA)
2860000 0000 0100 - 0001 0000 0000: main binary
2870001 0000 0000 - 0001 0000 0000: -
2880001 0000 0000 - 0b00 0000 0000: shadow
2890b00 0000 0000 - 0b00 0000 0000: -
2900b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
2910d00 0000 0000 - 0f00 0000 0000: -
2920f00 0000 0000 - 0f50 0000 0000: heap
2930f50 0000 0000 - 0f60 0000 0000: -
2940f60 0000 0000 - 1000 0000 0000: modules and main thread stack
295*/
296struct MappingPPC64_44 {
297 static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
298 kBrokenLinearity | kBrokenAliasedMetas;
299 static const uptr kMetaShadowBeg = 0x0b0000000000ull;
300 static const uptr kMetaShadowEnd = 0x0d0000000000ull;
301 static const uptr kShadowBeg = 0x000100000000ull;
302 static const uptr kShadowEnd = 0x0b0000000000ull;
303 static const uptr kLoAppMemBeg = 0x000000000100ull;
304 static const uptr kLoAppMemEnd = 0x000100000000ull;
305 static const uptr kHeapMemBeg = 0x0f0000000000ull;
306 static const uptr kHeapMemEnd = 0x0f5000000000ull;
307 static const uptr kHiAppMemBeg = 0x0f6000000000ull;
308 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
309 static const uptr kShadowMsk = 0x0f0000000000ull;
310 static const uptr kShadowXor = 0x002100000000ull;
311 static const uptr kShadowAdd = 0x000000000000ull;
312 static const uptr kVdsoBeg = 0x3c0000000000000ull;
313 static const uptr kMidAppMemBeg = 0;
314 static const uptr kMidAppMemEnd = 0;
315};
316
317/*
318C/C++ on linux/powerpc64 (46-bit VMA)
3190000 0000 1000 - 0100 0000 0000: main binary
3200100 0000 0000 - 0200 0000 0000: -
3210100 0000 0000 - 0800 0000 0000: shadow
3220800 0000 0000 - 1000 0000 0000: -
3231000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3241200 0000 0000 - 3d00 0000 0000: -
3253d00 0000 0000 - 3e00 0000 0000: heap
3263e00 0000 0000 - 3e80 0000 0000: -
3273e80 0000 0000 - 4000 0000 0000: modules and main thread stack
328*/
329struct MappingPPC64_46 {
330 static const uptr kMetaShadowBeg = 0x100000000000ull;
331 static const uptr kMetaShadowEnd = 0x120000000000ull;
332 static const uptr kShadowBeg = 0x010000000000ull;
333 static const uptr kShadowEnd = 0x080000000000ull;
334 static const uptr kHeapMemBeg = 0x3d0000000000ull;
335 static const uptr kHeapMemEnd = 0x3e0000000000ull;
336 static const uptr kLoAppMemBeg = 0x000000001000ull;
337 static const uptr kLoAppMemEnd = 0x010000000000ull;
338 static const uptr kHiAppMemBeg = 0x3e8000000000ull;
339 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
340 static const uptr kShadowMsk = 0x3c0000000000ull;
341 static const uptr kShadowXor = 0x020000000000ull;
342 static const uptr kShadowAdd = 0x000000000000ull;
343 static const uptr kVdsoBeg = 0x7800000000000000ull;
344 static const uptr kMidAppMemBeg = 0;
345 static const uptr kMidAppMemEnd = 0;
346};
347
348/*
349C/C++ on linux/powerpc64 (47-bit VMA)
3500000 0000 1000 - 0100 0000 0000: main binary
3510100 0000 0000 - 0200 0000 0000: -
3520100 0000 0000 - 0800 0000 0000: shadow
3530800 0000 0000 - 1000 0000 0000: -
3541000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
3551200 0000 0000 - 7d00 0000 0000: -
3567d00 0000 0000 - 7e00 0000 0000: heap
3577e00 0000 0000 - 7e80 0000 0000: -
3587e80 0000 0000 - 8000 0000 0000: modules and main thread stack
359*/
360struct MappingPPC64_47 {
361 static const uptr kMetaShadowBeg = 0x100000000000ull;
362 static const uptr kMetaShadowEnd = 0x120000000000ull;
363 static const uptr kShadowBeg = 0x010000000000ull;
364 static const uptr kShadowEnd = 0x080000000000ull;
365 static const uptr kHeapMemBeg = 0x7d0000000000ull;
366 static const uptr kHeapMemEnd = 0x7e0000000000ull;
367 static const uptr kLoAppMemBeg = 0x000000001000ull;
368 static const uptr kLoAppMemEnd = 0x010000000000ull;
369 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
370 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
371 static const uptr kShadowMsk = 0x7c0000000000ull;
372 static const uptr kShadowXor = 0x020000000000ull;
373 static const uptr kShadowAdd = 0x000000000000ull;
374 static const uptr kVdsoBeg = 0x7800000000000000ull;
375 static const uptr kMidAppMemBeg = 0;
376 static const uptr kMidAppMemEnd = 0;
377};
378
379/*
380C/C++ on linux/riscv64 (39-bit VMA)
3810000 0010 00 - 0200 0000 00: main binary ( 8 GB)
3820200 0000 00 - 1000 0000 00: -
3831000 0000 00 - 4000 0000 00: shadow memory (64 GB)
3844000 0000 00 - 4800 0000 00: metainfo (16 GB)
3854800 0000 00 - 5500 0000 00: -
3865500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB)
3875600 0000 00 - 7c00 0000 00: -
3887d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB)
389
390mmap by default allocates from top downwards
391VDSO sits below loader and above dynamic libraries, within HiApp region.
392Heap starts after program region whose position depends on pie or non-pie.
393Disable tracking them since their locations are not fixed.
394*/
395struct MappingRiscv64_39 {
396 static const uptr kLoAppMemBeg = 0x0000001000ull;
397 static const uptr kLoAppMemEnd = 0x0200000000ull;
398 static const uptr kShadowBeg = 0x1000000000ull;
399 static const uptr kShadowEnd = 0x2000000000ull;
400 static const uptr kMetaShadowBeg = 0x2000000000ull;
401 static const uptr kMetaShadowEnd = 0x2400000000ull;
402 static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
403 static const uptr kMidAppMemEnd = 0x2c00000000ull;
404 static const uptr kHeapMemBeg = 0x2c00000000ull;
405 static const uptr kHeapMemEnd = 0x2c00000000ull;
406 static const uptr kHiAppMemBeg = 0x3c00000000ull;
407 static const uptr kHiAppMemEnd = 0x3fffffffffull;
408 static const uptr kShadowMsk = 0x3800000000ull;
409 static const uptr kShadowXor = 0x0800000000ull;
410 static const uptr kShadowAdd = 0x0000000000ull;
411 static const uptr kVdsoBeg = 0x4000000000ull;
412};
413
414/*
415C/C++ on linux/riscv64 (48-bit VMA)
4160000 0000 1000 - 0400 0000 0000: main binary ( 4 TB)
4170500 0000 0000 - 2000 0000 0000: -
4182000 0000 0000 - 4000 0000 0000: shadow memory (32 TB)
4194000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB)
4204800 0000 0000 - 5555 5555 5000: -
4215555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB)
4225a00 0000 0000 - 7a00 0000 0000: -
4237a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB)
424*/
425struct MappingRiscv64_48 {
426 static const uptr kLoAppMemBeg = 0x000000001000ull;
427 static const uptr kLoAppMemEnd = 0x040000000000ull;
428 static const uptr kShadowBeg = 0x200000000000ull;
429 static const uptr kShadowEnd = 0x400000000000ull;
430 static const uptr kMetaShadowBeg = 0x400000000000ull;
431 static const uptr kMetaShadowEnd = 0x480000000000ull;
432 static const uptr kMidAppMemBeg = 0x555555555000ull;
433 static const uptr kMidAppMemEnd = 0x5a0000000000ull;
434 static const uptr kHeapMemBeg = 0x5a0000000000ull;
435 static const uptr kHeapMemEnd = 0x5a0000000000ull;
436 static const uptr kHiAppMemBeg = 0x7a0000000000ull;
437 static const uptr kHiAppMemEnd = 0x7fffffffffffull;
438 static const uptr kShadowMsk = 0x700000000000ull;
439 static const uptr kShadowXor = 0x100000000000ull;
440 static const uptr kShadowAdd = 0x000000000000ull;
441 static const uptr kVdsoBeg = 0x800000000000ull;
442};
443
444/*
445C/C++ on linux/s390x
446While the kernel provides a 64-bit address space, we have to restrict ourselves
447to 48 bits due to how e.g. SyncVar::GetId() works.
4480000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
4490e00 0000 0000 - 2000 0000 0000: -
4502000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
4514000 0000 0000 - 9000 0000 0000: -
4529000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
4539800 0000 0000 - be00 0000 0000: -
454be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
455*/
456struct MappingS390x {
457 static const uptr kMetaShadowBeg = 0x900000000000ull;
458 static const uptr kMetaShadowEnd = 0x980000000000ull;
459 static const uptr kShadowBeg = 0x200000000000ull;
460 static const uptr kShadowEnd = 0x400000000000ull;
461 static const uptr kHeapMemBeg = 0xbe0000000000ull;
462 static const uptr kHeapMemEnd = 0xc00000000000ull;
463 static const uptr kLoAppMemBeg = 0x000000001000ull;
464 static const uptr kLoAppMemEnd = 0x0e0000000000ull;
465 static const uptr kHiAppMemBeg = 0xc00000004000ull;
466 static const uptr kHiAppMemEnd = 0xc00000004000ull;
467 static const uptr kShadowMsk = 0xb00000000000ull;
468 static const uptr kShadowXor = 0x100000000000ull;
469 static const uptr kShadowAdd = 0x000000000000ull;
470 static const uptr kVdsoBeg = 0xfffffffff000ull;
471 static const uptr kMidAppMemBeg = 0;
472 static const uptr kMidAppMemEnd = 0;
473};
474
475/* Go on linux, darwin and freebsd on x86_64
4760000 0000 1000 - 0000 1000 0000: executable
4770000 1000 0000 - 00c0 0000 0000: -
47800c0 0000 0000 - 00e0 0000 0000: heap
47900e0 0000 0000 - 2000 0000 0000: -
4802000 0000 0000 - 21c0 0000 0000: shadow
48121c0 0000 0000 - 3000 0000 0000: -
4823000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
4834000 0000 0000 - 8000 0000 0000: -
484*/
485
486struct MappingGo48 {
487 static const uptr kMetaShadowBeg = 0x300000000000ull;
488 static const uptr kMetaShadowEnd = 0x400000000000ull;
489 static const uptr kShadowBeg = 0x200000000000ull;
490 static const uptr kShadowEnd = 0x21c000000000ull;
491 static const uptr kLoAppMemBeg = 0x000000001000ull;
492 static const uptr kLoAppMemEnd = 0x00e000000000ull;
493 static const uptr kMidAppMemBeg = 0;
494 static const uptr kMidAppMemEnd = 0;
495 static const uptr kHiAppMemBeg = 0;
496 static const uptr kHiAppMemEnd = 0;
497 static const uptr kHeapMemBeg = 0;
498 static const uptr kHeapMemEnd = 0;
499 static const uptr kVdsoBeg = 0;
500 static const uptr kShadowMsk = 0;
501 static const uptr kShadowXor = 0;
502 static const uptr kShadowAdd = 0x200000000000ull;
503};
504
505/* Go on windows
5060000 0000 1000 - 0000 1000 0000: executable
5070000 1000 0000 - 00f8 0000 0000: -
50800c0 0000 0000 - 00e0 0000 0000: heap
50900e0 0000 0000 - 0100 0000 0000: -
5100100 0000 0000 - 0300 0000 0000: shadow
5110300 0000 0000 - 0700 0000 0000: -
5120700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
51307d0 0000 0000 - 8000 0000 0000: -
514PIE binaries currently not supported, but it should be theoretically possible.
515*/
516
517struct MappingGoWindows {
518 static const uptr kMetaShadowBeg = 0x070000000000ull;
519 static const uptr kMetaShadowEnd = 0x077000000000ull;
520 static const uptr kShadowBeg = 0x010000000000ull;
521 static const uptr kShadowEnd = 0x030000000000ull;
522 static const uptr kLoAppMemBeg = 0x000000001000ull;
523 static const uptr kLoAppMemEnd = 0x00e000000000ull;
524 static const uptr kMidAppMemBeg = 0;
525 static const uptr kMidAppMemEnd = 0;
526 static const uptr kHiAppMemBeg = 0;
527 static const uptr kHiAppMemEnd = 0;
528 static const uptr kHeapMemBeg = 0;
529 static const uptr kHeapMemEnd = 0;
530 static const uptr kVdsoBeg = 0;
531 static const uptr kShadowMsk = 0;
532 static const uptr kShadowXor = 0;
533 static const uptr kShadowAdd = 0x010000000000ull;
534};
535
536/* Go on linux/powerpc64 (46-bit VMA)
5370000 0000 1000 - 0000 1000 0000: executable
5380000 1000 0000 - 00c0 0000 0000: -
53900c0 0000 0000 - 00e0 0000 0000: heap
54000e0 0000 0000 - 2000 0000 0000: -
5412000 0000 0000 - 21c0 0000 0000: shadow
54221c0 0000 0000 - 2400 0000 0000: -
5432400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
5442470 0000 0000 - 4000 0000 0000: -
545*/
546
547struct MappingGoPPC64_46 {
548 static const uptr kMetaShadowBeg = 0x240000000000ull;
549 static const uptr kMetaShadowEnd = 0x247000000000ull;
550 static const uptr kShadowBeg = 0x200000000000ull;
551 static const uptr kShadowEnd = 0x21c000000000ull;
552 static const uptr kLoAppMemBeg = 0x000000001000ull;
553 static const uptr kLoAppMemEnd = 0x00e000000000ull;
554 static const uptr kMidAppMemBeg = 0;
555 static const uptr kMidAppMemEnd = 0;
556 static const uptr kHiAppMemBeg = 0;
557 static const uptr kHiAppMemEnd = 0;
558 static const uptr kHeapMemBeg = 0;
559 static const uptr kHeapMemEnd = 0;
560 static const uptr kVdsoBeg = 0;
561 static const uptr kShadowMsk = 0;
562 static const uptr kShadowXor = 0;
563 static const uptr kShadowAdd = 0x200000000000ull;
564};
565
566/* Go on linux/powerpc64 (47-bit VMA)
5670000 0000 1000 - 0000 1000 0000: executable
5680000 1000 0000 - 00c0 0000 0000: -
56900c0 0000 0000 - 00e0 0000 0000: heap
57000e0 0000 0000 - 2000 0000 0000: -
5712000 0000 0000 - 2800 0000 0000: shadow
5722800 0000 0000 - 3000 0000 0000: -
5733000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
5743200 0000 0000 - 8000 0000 0000: -
575*/
576
577struct MappingGoPPC64_47 {
578 static const uptr kMetaShadowBeg = 0x300000000000ull;
579 static const uptr kMetaShadowEnd = 0x320000000000ull;
580 static const uptr kShadowBeg = 0x200000000000ull;
581 static const uptr kShadowEnd = 0x280000000000ull;
582 static const uptr kLoAppMemBeg = 0x000000001000ull;
583 static const uptr kLoAppMemEnd = 0x00e000000000ull;
584 static const uptr kMidAppMemBeg = 0;
585 static const uptr kMidAppMemEnd = 0;
586 static const uptr kHiAppMemBeg = 0;
587 static const uptr kHiAppMemEnd = 0;
588 static const uptr kHeapMemBeg = 0;
589 static const uptr kHeapMemEnd = 0;
590 static const uptr kVdsoBeg = 0;
591 static const uptr kShadowMsk = 0;
592 static const uptr kShadowXor = 0;
593 static const uptr kShadowAdd = 0x200000000000ull;
594};
595
596/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
5970000 0000 1000 - 0000 1000 0000: executable
5980000 1000 0000 - 00c0 0000 0000: -
59900c0 0000 0000 - 00e0 0000 0000: heap
60000e0 0000 0000 - 2000 0000 0000: -
6012000 0000 0000 - 2800 0000 0000: shadow
6022800 0000 0000 - 3000 0000 0000: -
6033000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6043200 0000 0000 - 8000 0000 0000: -
605*/
606struct MappingGoAarch64 {
607 static const uptr kMetaShadowBeg = 0x300000000000ull;
608 static const uptr kMetaShadowEnd = 0x320000000000ull;
609 static const uptr kShadowBeg = 0x200000000000ull;
610 static const uptr kShadowEnd = 0x280000000000ull;
611 static const uptr kLoAppMemBeg = 0x000000001000ull;
612 static const uptr kLoAppMemEnd = 0x00e000000000ull;
613 static const uptr kMidAppMemBeg = 0;
614 static const uptr kMidAppMemEnd = 0;
615 static const uptr kHiAppMemBeg = 0;
616 static const uptr kHiAppMemEnd = 0;
617 static const uptr kHeapMemBeg = 0;
618 static const uptr kHeapMemEnd = 0;
619 static const uptr kVdsoBeg = 0;
620 static const uptr kShadowMsk = 0;
621 static const uptr kShadowXor = 0;
622 static const uptr kShadowAdd = 0x200000000000ull;
623};
624
625/* Go on linux/loongarch64 (47-bit VMA)
6260000 0000 1000 - 0000 1000 0000: executable
6270000 1000 0000 - 00c0 0000 0000: -
62800c0 0000 0000 - 00e0 0000 0000: heap
62900e0 0000 0000 - 2000 0000 0000: -
6302000 0000 0000 - 2800 0000 0000: shadow
6312800 0000 0000 - 3000 0000 0000: -
6323000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6333200 0000 0000 - 8000 0000 0000: -
634*/
635struct MappingGoLoongArch64_47 {
636 static const uptr kMetaShadowBeg = 0x300000000000ull;
637 static const uptr kMetaShadowEnd = 0x320000000000ull;
638 static const uptr kShadowBeg = 0x200000000000ull;
639 static const uptr kShadowEnd = 0x280000000000ull;
640 static const uptr kLoAppMemBeg = 0x000000001000ull;
641 static const uptr kLoAppMemEnd = 0x00e000000000ull;
642 static const uptr kMidAppMemBeg = 0;
643 static const uptr kMidAppMemEnd = 0;
644 static const uptr kHiAppMemBeg = 0;
645 static const uptr kHiAppMemEnd = 0;
646 static const uptr kHeapMemBeg = 0;
647 static const uptr kHeapMemEnd = 0;
648 static const uptr kVdsoBeg = 0;
649 static const uptr kShadowMsk = 0;
650 static const uptr kShadowXor = 0;
651 static const uptr kShadowAdd = 0x200000000000ull;
652};
653
654/*
655Go on linux/mips64 (47-bit VMA)
6560000 0000 1000 - 0000 1000 0000: executable
6570000 1000 0000 - 00c0 0000 0000: -
65800c0 0000 0000 - 00e0 0000 0000: heap
65900e0 0000 0000 - 2000 0000 0000: -
6602000 0000 0000 - 2800 0000 0000: shadow
6612800 0000 0000 - 3000 0000 0000: -
6623000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
6633200 0000 0000 - 8000 0000 0000: -
664*/
665struct MappingGoMips64_47 {
666 static const uptr kMetaShadowBeg = 0x300000000000ull;
667 static const uptr kMetaShadowEnd = 0x320000000000ull;
668 static const uptr kShadowBeg = 0x200000000000ull;
669 static const uptr kShadowEnd = 0x280000000000ull;
670 static const uptr kLoAppMemBeg = 0x000000001000ull;
671 static const uptr kLoAppMemEnd = 0x00e000000000ull;
672 static const uptr kMidAppMemBeg = 0;
673 static const uptr kMidAppMemEnd = 0;
674 static const uptr kHiAppMemBeg = 0;
675 static const uptr kHiAppMemEnd = 0;
676 static const uptr kHeapMemBeg = 0;
677 static const uptr kHeapMemEnd = 0;
678 static const uptr kVdsoBeg = 0;
679 static const uptr kShadowMsk = 0;
680 static const uptr kShadowXor = 0;
681 static const uptr kShadowAdd = 0x200000000000ull;
682};
683
684/* Go on linux/riscv64 (48-bit VMA)
6850000 0001 0000 - 00e0 0000 0000: executable and heap (896 GiB)
68600e0 0000 0000 - 2000 0000 0000: -
6872000 0000 0000 - 2400 0000 0000: shadow - 4 TiB ( ~ 4 * app)
6882400 0000 0000 - 3000 0000 0000: -
6893000 0000 0000 - 3100 0000 0000: metainfo - 1 TiB ( ~ 1 * app)
6903100 0000 0000 - 8000 0000 0000: -
691*/
692struct MappingGoRiscv64 {
693 static const uptr kMetaShadowBeg = 0x300000000000ull;
694 static const uptr kMetaShadowEnd = 0x310000000000ull;
695 static const uptr kShadowBeg = 0x200000000000ull;
696 static const uptr kShadowEnd = 0x240000000000ull;
697 static const uptr kLoAppMemBeg = 0x000000010000ull;
698 static const uptr kLoAppMemEnd = 0x000e00000000ull;
699 static const uptr kMidAppMemBeg = 0;
700 static const uptr kMidAppMemEnd = 0;
701 static const uptr kHiAppMemBeg = 0;
702 static const uptr kHiAppMemEnd = 0;
703 static const uptr kHeapMemBeg = 0;
704 static const uptr kHeapMemEnd = 0;
705 static const uptr kVdsoBeg = 0;
706 static const uptr kShadowMsk = 0;
707 static const uptr kShadowXor = 0;
708 static const uptr kShadowAdd = 0x200000000000ull;
709};
710
711/*
712Go on linux/s390x
7130000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
7141000 0000 0000 - 4000 0000 0000: -
7154000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
7166000 0000 0000 - 9000 0000 0000: -
7179000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
718*/
719struct MappingGoS390x {
720 static const uptr kMetaShadowBeg = 0x900000000000ull;
721 static const uptr kMetaShadowEnd = 0x980000000000ull;
722 static const uptr kShadowBeg = 0x400000000000ull;
723 static const uptr kShadowEnd = 0x600000000000ull;
724 static const uptr kLoAppMemBeg = 0x000000001000ull;
725 static const uptr kLoAppMemEnd = 0x100000000000ull;
726 static const uptr kMidAppMemBeg = 0;
727 static const uptr kMidAppMemEnd = 0;
728 static const uptr kHiAppMemBeg = 0;
729 static const uptr kHiAppMemEnd = 0;
730 static const uptr kHeapMemBeg = 0;
731 static const uptr kHeapMemEnd = 0;
732 static const uptr kVdsoBeg = 0;
733 static const uptr kShadowMsk = 0;
734 static const uptr kShadowXor = 0;
735 static const uptr kShadowAdd = 0x400000000000ull;
736};
737
738extern uptr vmaSize;
739
740template <typename Func, typename Arg>
741ALWAYS_INLINE auto SelectMapping(Arg arg) {
742#if SANITIZER_GO
743# if defined(__powerpc64__)
744 switch (vmaSize) {
745 case 46:
746 return Func::template Apply<MappingGoPPC64_46>(arg);
747 case 47:
748 return Func::template Apply<MappingGoPPC64_47>(arg);
749 }
750# elif defined(__mips64)
751 return Func::template Apply<MappingGoMips64_47>(arg);
752# elif defined(__s390x__)
753 return Func::template Apply<MappingGoS390x>(arg);
754# elif defined(__aarch64__)
755 return Func::template Apply<MappingGoAarch64>(arg);
756# elif defined(__loongarch_lp64)
757 return Func::template Apply<MappingGoLoongArch64_47>(arg);
758# elif SANITIZER_RISCV64
759 return Func::template Apply<MappingGoRiscv64>(arg);
760# elif SANITIZER_WINDOWS
761 return Func::template Apply<MappingGoWindows>(arg);
762# else
763 return Func::template Apply<MappingGo48>(arg);
764# endif
765#else // SANITIZER_GO
766# if SANITIZER_IOS && !SANITIZER_IOSSIM
767 return Func::template Apply<MappingAppleAarch64>(arg);
768# elif defined(__x86_64__) || SANITIZER_APPLE
769 return Func::template Apply<Mapping48AddressSpace>(arg);
770# elif defined(__aarch64__)
771 switch (vmaSize) {
772 case 39:
773 return Func::template Apply<MappingAarch64_39>(arg);
774 case 42:
775 return Func::template Apply<MappingAarch64_42>(arg);
776 case 48:
777 return Func::template Apply<MappingAarch64_48>(arg);
778 }
779# elif SANITIZER_LOONGARCH64
780 return Func::template Apply<MappingLoongArch64_47>(arg);
781# elif defined(__powerpc64__)
782 switch (vmaSize) {
783 case 44:
784 return Func::template Apply<MappingPPC64_44>(arg);
785 case 46:
786 return Func::template Apply<MappingPPC64_46>(arg);
787 case 47:
788 return Func::template Apply<MappingPPC64_47>(arg);
789 }
790# elif defined(__mips64)
791 return Func::template Apply<MappingMips64_40>(arg);
792# elif SANITIZER_RISCV64
793 switch (vmaSize) {
794 case 39:
795 return Func::template Apply<MappingRiscv64_39>(arg);
796 case 48:
797 return Func::template Apply<MappingRiscv64_48>(arg);
798 }
799# elif defined(__s390x__)
800 return Func::template Apply<MappingS390x>(arg);
801# else
802# error "unsupported platform"
803# endif
804#endif
805 Die();
806}
807
808template <typename Func>
809void ForEachMapping() {
810 Func::template Apply<Mapping48AddressSpace>();
811 Func::template Apply<MappingMips64_40>();
812 Func::template Apply<MappingAppleAarch64>();
813 Func::template Apply<MappingAarch64_39>();
814 Func::template Apply<MappingAarch64_42>();
815 Func::template Apply<MappingAarch64_48>();
816 Func::template Apply<MappingLoongArch64_47>();
817 Func::template Apply<MappingPPC64_44>();
818 Func::template Apply<MappingPPC64_46>();
819 Func::template Apply<MappingPPC64_47>();
820 Func::template Apply<MappingRiscv64_39>();
821 Func::template Apply<MappingRiscv64_48>();
822 Func::template Apply<MappingS390x>();
823 Func::template Apply<MappingGo48>();
824 Func::template Apply<MappingGoWindows>();
825 Func::template Apply<MappingGoPPC64_46>();
826 Func::template Apply<MappingGoPPC64_47>();
827 Func::template Apply<MappingGoAarch64>();
828 Func::template Apply<MappingGoLoongArch64_47>();
829 Func::template Apply<MappingGoMips64_47>();
830 Func::template Apply<MappingGoRiscv64>();
831 Func::template Apply<MappingGoS390x>();
832}
833
834enum MappingType {
835 kLoAppMemBeg,
836 kLoAppMemEnd,
837 kHiAppMemBeg,
838 kHiAppMemEnd,
839 kMidAppMemBeg,
840 kMidAppMemEnd,
841 kHeapMemBeg,
842 kHeapMemEnd,
843 kShadowBeg,
844 kShadowEnd,
845 kMetaShadowBeg,
846 kMetaShadowEnd,
847 kVdsoBeg,
848};
849
850struct MappingField {
851 template <typename Mapping>
852 static uptr Apply(MappingType type) {
853 switch (type) {
854 case kLoAppMemBeg:
855 return Mapping::kLoAppMemBeg;
856 case kLoAppMemEnd:
857 return Mapping::kLoAppMemEnd;
858 case kMidAppMemBeg:
859 return Mapping::kMidAppMemBeg;
860 case kMidAppMemEnd:
861 return Mapping::kMidAppMemEnd;
862 case kHiAppMemBeg:
863 return Mapping::kHiAppMemBeg;
864 case kHiAppMemEnd:
865 return Mapping::kHiAppMemEnd;
866 case kHeapMemBeg:
867 return Mapping::kHeapMemBeg;
868 case kHeapMemEnd:
869 return Mapping::kHeapMemEnd;
870 case kVdsoBeg:
871 return Mapping::kVdsoBeg;
872 case kShadowBeg:
873 return Mapping::kShadowBeg;
874 case kShadowEnd:
875 return Mapping::kShadowEnd;
876 case kMetaShadowBeg:
877 return Mapping::kMetaShadowBeg;
878 case kMetaShadowEnd:
879 return Mapping::kMetaShadowEnd;
880 }
881 Die();
882 }
883};
884
885ALWAYS_INLINE
886uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
887ALWAYS_INLINE
888uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
889
890ALWAYS_INLINE
891uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
892ALWAYS_INLINE
893uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
894
895ALWAYS_INLINE
896uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
897ALWAYS_INLINE
898uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
899
900ALWAYS_INLINE
901uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
902ALWAYS_INLINE
903uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
904
905ALWAYS_INLINE
906uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
907
908ALWAYS_INLINE
909uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
910ALWAYS_INLINE
911uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
912
913ALWAYS_INLINE
914uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
915ALWAYS_INLINE
916uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
917
918struct IsAppMemImpl {
919 template <typename Mapping>
920 static bool Apply(uptr mem) {
921 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
922 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
923 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
924 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
925 }
926};
927
928ALWAYS_INLINE
929bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
930
931struct IsShadowMemImpl {
932 template <typename Mapping>
933 static bool Apply(uptr mem) {
934 return mem >= Mapping::kShadowBeg && mem < Mapping::kShadowEnd;
935 }
936};
937
938ALWAYS_INLINE
939bool IsShadowMem(RawShadow *p) {
940 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
941}
942
943struct IsMetaMemImpl {
944 template <typename Mapping>
945 static bool Apply(uptr mem) {
946 return mem >= Mapping::kMetaShadowBeg && mem < Mapping::kMetaShadowEnd;
947 }
948};
949
950ALWAYS_INLINE
951bool IsMetaMem(const u32 *p) {
952 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
953}
954
955struct MemToShadowImpl {
956 template <typename Mapping>
957 static uptr Apply(uptr x) {
958 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
959 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
960 Mapping::kShadowXor) *
961 kShadowMultiplier +
962 Mapping::kShadowAdd;
963 }
964};
965
966ALWAYS_INLINE
967RawShadow *MemToShadow(uptr x) {
968 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
969}
970
971struct MemToMetaImpl {
972 template <typename Mapping>
973 static u32 *Apply(uptr x) {
974 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
975 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
976 kMetaShadowCell * kMetaShadowSize) |
977 Mapping::kMetaShadowBeg);
978 }
979};
980
981ALWAYS_INLINE
982u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
983
984struct ShadowToMemImpl {
985 template <typename Mapping>
986 static uptr Apply(uptr sp) {
987 if (!IsShadowMemImpl::Apply<Mapping>(sp))
988 return 0;
989 // The shadow mapping is non-linear and we've lost some bits, so we don't
990 // have an easy way to restore the original app address. But the mapping is
991 // a bijection, so we try to restore the address as belonging to
992 // low/mid/high range consecutively and see if shadow->app->shadow mapping
993 // gives us the same address.
994 uptr p =
995 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
996 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
997 MemToShadowImpl::Apply<Mapping>(p) == sp)
998 return p;
999 if (Mapping::kMidAppMemBeg) {
1000 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
1001 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
1002 MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
1003 return p_mid;
1004 }
1005 return p | Mapping::kShadowMsk;
1006 }
1007};
1008
1009ALWAYS_INLINE
1010uptr ShadowToMem(RawShadow *s) {
1011 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
1012}
1013
1014// Compresses addr to kCompressedAddrBits stored in least significant bits.
1015ALWAYS_INLINE uptr CompressAddr(uptr addr) {
1016 return addr & ((1ull << kCompressedAddrBits) - 1);
1017}
1018
1019struct RestoreAddrImpl {
1020 typedef uptr Result;
1021 template <typename Mapping>
1022 static Result Apply(uptr addr) {
1023 // To restore the address we go over all app memory ranges and check if top
1024 // 3 bits of the compressed addr match that of the app range. If yes, we
1025 // assume that the compressed address come from that range and restore the
1026 // missing top bits to match the app range address.
1027 const uptr ranges[] = {
1028 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
1029 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
1030 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
1031 };
1032 const uptr indicator = 0x0e0000000000ull;
1033 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
1034 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
1035 uptr beg = ranges[i];
1036 uptr end = ranges[i + 1];
1037 if (beg == end)
1038 continue;
1039 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
1040 if ((addr & indicator) == (p & indicator))
1041 return addr | (p & ~(ind_lsb - 1));
1042 }
1043 }
1044 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
1045 Die();
1046 }
1047};
1048
1049// Restores compressed addr from kCompressedAddrBits to full representation.
1050// This is called only during reporting and is not performance-critical.
1051inline uptr RestoreAddr(uptr addr) {
1052 return SelectMapping<RestoreAddrImpl>(addr);
1053}
1054
1055void InitializePlatform();
1056void InitializePlatformEarly();
1057bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
1058void InitializeShadowMemoryPlatform();
1059void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1060int ExtractResolvFDs(void *state, int *fds, int nfd);
1061int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
1062uptr ExtractLongJmpSp(uptr *env);
1063void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
1064
1065int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
1066 void (*cleanup)(void *arg), void *arg);
1067
1068void DestroyThreadState();
1069void PlatformCleanUpThreadState(ThreadState *thr);
1070
1071} // namespace __tsan
1072
1073#endif // TSAN_PLATFORM_H