1//===-- tsan_vector_clock.cpp ---------------------------------------------===//
  2//
  3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4// See https://llvm.org/LICENSE.txt for license information.
  5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6//
  7//===----------------------------------------------------------------------===//
  8//
  9// This file is a part of ThreadSanitizer (TSan), a race detector.
 10//
 11//===----------------------------------------------------------------------===//
 12#include "tsan_vector_clock.h"
 13
 14#include "sanitizer_common/sanitizer_placement_new.h"
 15#include "tsan_mman.h"
 16
 17namespace __tsan {
 18
 19#if TSAN_VECTORIZE
 20const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
 21#endif
 22
 23VectorClock::VectorClock() { Reset(); }
 24
 25void VectorClock::Reset() {
 26#if !TSAN_VECTORIZE
 27  for (uptr i = 0; i < kThreadSlotCount; i++)
 28    clk_[i] = kEpochZero;
 29#else
 30  m128 z = _mm_setzero_si128();
 31  m128* vclk = reinterpret_cast<m128*>(clk_);
 32  for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
 33#endif
 34}
 35
 36void VectorClock::Acquire(const VectorClock* src) {
 37  if (!src)
 38    return;
 39#if !TSAN_VECTORIZE
 40  for (uptr i = 0; i < kThreadSlotCount; i++)
 41    clk_[i] = max(clk_[i], src->clk_[i]);
 42#else
 43  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
 44  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
 45  for (uptr i = 0; i < kVectorClockSize; i++) {
 46    m128 s = _mm_load_si128(&vsrc[i]);
 47    m128 d = _mm_load_si128(&vdst[i]);
 48    m128 m = _mm_max_epu16(s, d);
 49    _mm_store_si128(&vdst[i], m);
 50  }
 51#endif
 52}
 53
 54static VectorClock* AllocClock(VectorClock** dstp) {
 55  if (UNLIKELY(!*dstp))
 56    *dstp = New<VectorClock>();
 57  return *dstp;
 58}
 59
 60void VectorClock::Release(VectorClock** dstp) const {
 61  VectorClock* dst = AllocClock(dstp);
 62  dst->Acquire(this);
 63}
 64
 65void VectorClock::ReleaseStore(VectorClock** dstp) const {
 66  VectorClock* dst = AllocClock(dstp);
 67  *dst = *this;
 68}
 69
 70VectorClock& VectorClock::operator=(const VectorClock& other) {
 71#if !TSAN_VECTORIZE
 72  for (uptr i = 0; i < kThreadSlotCount; i++)
 73    clk_[i] = other.clk_[i];
 74#else
 75  m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
 76  m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
 77  for (uptr i = 0; i < kVectorClockSize; i++) {
 78    m128 s = _mm_load_si128(&vsrc[i]);
 79    _mm_store_si128(&vdst[i], s);
 80  }
 81#endif
 82  return *this;
 83}
 84
 85void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
 86  VectorClock* dst = AllocClock(dstp);
 87#if !TSAN_VECTORIZE
 88  for (uptr i = 0; i < kThreadSlotCount; i++) {
 89    Epoch tmp = dst->clk_[i];
 90    dst->clk_[i] = clk_[i];
 91    clk_[i] = max(clk_[i], tmp);
 92  }
 93#else
 94  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
 95  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
 96  for (uptr i = 0; i < kVectorClockSize; i++) {
 97    m128 t = _mm_load_si128(&vdst[i]);
 98    m128 c = _mm_load_si128(&vclk[i]);
 99    m128 m = _mm_max_epu16(c, t);
100    _mm_store_si128(&vdst[i], c);
101    _mm_store_si128(&vclk[i], m);
102  }
103#endif
104}
105
106void VectorClock::ReleaseAcquire(VectorClock** dstp) {
107  VectorClock* dst = AllocClock(dstp);
108#if !TSAN_VECTORIZE
109  for (uptr i = 0; i < kThreadSlotCount; i++) {
110    dst->clk_[i] = max(dst->clk_[i], clk_[i]);
111    clk_[i] = dst->clk_[i];
112  }
113#else
114  m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
115  m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
116  for (uptr i = 0; i < kVectorClockSize; i++) {
117    m128 c = _mm_load_si128(&vclk[i]);
118    m128 d = _mm_load_si128(&vdst[i]);
119    m128 m = _mm_max_epu16(c, d);
120    _mm_store_si128(&vdst[i], m);
121    _mm_store_si128(&vclk[i], m);
122  }
123#endif
124}
125
126}  // namespace __tsan