1//===----------------------------------------------------------------------===//
  2//
  3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  4// See https://llvm.org/LICENSE.txt for license information.
  5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  6//
  7//===----------------------------------------------------------------------===//
  8
  9#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
 10#define _LIBCPP___ATOMIC_SUPPORT_C11_H
 11
 12#include <__atomic/memory_order.h>
 13#include <__config>
 14#include <__cstddef/ptrdiff_t.h>
 15#include <__memory/addressof.h>
 16#include <__type_traits/remove_const.h>
 17
 18#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 19#  pragma GCC system_header
 20#endif
 21
 22//
 23// This file implements support for C11-style atomics
 24//
 25
 26_LIBCPP_BEGIN_NAMESPACE_STD
 27
 28template <typename _Tp>
 29struct __cxx_atomic_base_impl {
 30  _LIBCPP_HIDE_FROM_ABI
 31#ifndef _LIBCPP_CXX03_LANG
 32  __cxx_atomic_base_impl() _NOEXCEPT = default;
 33#else
 34  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
 35  }
 36#endif // _LIBCPP_CXX03_LANG
 37  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
 38  _Atomic(_Tp) __a_value;
 39};
 40
 41#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s)
 42
 43_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT {
 44  __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order));
 45}
 46
 47_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT {
 48  __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));
 49}
 50
 51template <class _Tp>
 52_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
 53  __c11_atomic_init(std::addressof(__a->__a_value), __val);
 54}
 55template <class _Tp>
 56_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
 57  __c11_atomic_init(std::addressof(__a->__a_value), __val);
 58}
 59
 60template <class _Tp>
 61_LIBCPP_HIDE_FROM_ABI void
 62__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
 63  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 64}
 65template <class _Tp>
 66_LIBCPP_HIDE_FROM_ABI void
 67__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
 68  __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 69}
 70
 71template <class _Tp>
 72_LIBCPP_HIDE_FROM_ABI _Tp
 73__cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT {
 74  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 75  return __c11_atomic_load(
 76      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 77}
 78template <class _Tp>
 79_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT {
 80  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 81  return __c11_atomic_load(
 82      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 83}
 84
 85template <class _Tp>
 86_LIBCPP_HIDE_FROM_ABI void
 87__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
 88  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 89  *__dst           = __c11_atomic_load(
 90      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 91}
 92template <class _Tp>
 93_LIBCPP_HIDE_FROM_ABI void
 94__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
 95  using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
 96  *__dst           = __c11_atomic_load(
 97      const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 98}
 99
100template <class _Tp>
101_LIBCPP_HIDE_FROM_ABI _Tp
102__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
103  return __c11_atomic_exchange(
104      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
105}
106template <class _Tp>
107_LIBCPP_HIDE_FROM_ABI _Tp
108__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
109  return __c11_atomic_exchange(
110      std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
111}
112
113_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR memory_order __to_failure_order(memory_order __order) {
114  // Avoid switch statement to make this a constexpr.
115  return __order == memory_order_release
116           ? memory_order_relaxed
117           : (__order == memory_order_acq_rel ? memory_order_acquire : __order);
118}
119
120template <class _Tp>
121_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
122    __cxx_atomic_base_impl<_Tp> volatile* __a,
123    _Tp* __expected,
124    _Tp __value,
125    memory_order __success,
126    memory_order __failure) _NOEXCEPT {
127  return __c11_atomic_compare_exchange_strong(
128      std::addressof(__a->__a_value),
129      __expected,
130      __value,
131      static_cast<__memory_order_underlying_t>(__success),
132      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
133}
134template <class _Tp>
135_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
136    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
137    _NOEXCEPT {
138  return __c11_atomic_compare_exchange_strong(
139      std::addressof(__a->__a_value),
140      __expected,
141      __value,
142      static_cast<__memory_order_underlying_t>(__success),
143      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
144}
145
146template <class _Tp>
147_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
148    __cxx_atomic_base_impl<_Tp> volatile* __a,
149    _Tp* __expected,
150    _Tp __value,
151    memory_order __success,
152    memory_order __failure) _NOEXCEPT {
153  return __c11_atomic_compare_exchange_weak(
154      std::addressof(__a->__a_value),
155      __expected,
156      __value,
157      static_cast<__memory_order_underlying_t>(__success),
158      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
159}
160template <class _Tp>
161_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
162    __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
163    _NOEXCEPT {
164  return __c11_atomic_compare_exchange_weak(
165      std::addressof(__a->__a_value),
166      __expected,
167      __value,
168      static_cast<__memory_order_underlying_t>(__success),
169      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
170}
171
172template <class _Tp>
173_LIBCPP_HIDE_FROM_ABI _Tp
174__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
175  return __c11_atomic_fetch_add(
176      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
177}
178template <class _Tp>
179_LIBCPP_HIDE_FROM_ABI _Tp
180__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
181  return __c11_atomic_fetch_add(
182      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
183}
184
185template <class _Tp>
186_LIBCPP_HIDE_FROM_ABI _Tp*
187__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
188  return __c11_atomic_fetch_add(
189      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
190}
191template <class _Tp>
192_LIBCPP_HIDE_FROM_ABI _Tp*
193__cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
194  return __c11_atomic_fetch_add(
195      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
196}
197
198template <class _Tp>
199_LIBCPP_HIDE_FROM_ABI _Tp
200__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
201  return __c11_atomic_fetch_sub(
202      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
203}
204template <class _Tp>
205_LIBCPP_HIDE_FROM_ABI _Tp
206__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Tp __delta, memory_order __order) _NOEXCEPT {
207  return __c11_atomic_fetch_sub(
208      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
209}
210template <class _Tp>
211_LIBCPP_HIDE_FROM_ABI _Tp*
212__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
213  return __c11_atomic_fetch_sub(
214      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
215}
216template <class _Tp>
217_LIBCPP_HIDE_FROM_ABI _Tp*
218__cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*>* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT {
219  return __c11_atomic_fetch_sub(
220      std::addressof(__a->__a_value), __delta, static_cast<__memory_order_underlying_t>(__order));
221}
222
223template <class _Tp>
224_LIBCPP_HIDE_FROM_ABI _Tp
225__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
226  return __c11_atomic_fetch_and(
227      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
228}
229template <class _Tp>
230_LIBCPP_HIDE_FROM_ABI _Tp
231__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
232  return __c11_atomic_fetch_and(
233      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
234}
235
236template <class _Tp>
237_LIBCPP_HIDE_FROM_ABI _Tp
238__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
239  return __c11_atomic_fetch_or(
240      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
241}
242template <class _Tp>
243_LIBCPP_HIDE_FROM_ABI _Tp
244__cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
245  return __c11_atomic_fetch_or(
246      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
247}
248
249template <class _Tp>
250_LIBCPP_HIDE_FROM_ABI _Tp
251__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
252  return __c11_atomic_fetch_xor(
253      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
254}
255template <class _Tp>
256_LIBCPP_HIDE_FROM_ABI _Tp
257__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) _NOEXCEPT {
258  return __c11_atomic_fetch_xor(
259      std::addressof(__a->__a_value), __pattern, static_cast<__memory_order_underlying_t>(__order));
260}
261
262_LIBCPP_END_NAMESPACE_STD
263
264#endif // _LIBCPP___ATOMIC_SUPPORT_C11_H