1/*	$NetBSD: lock.h,v 1.29 2022/02/12 17:17:54 riastradh Exp $	*/
  2
  3/*-
  4 * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
  5 * All rights reserved.
  6 *
  7 * This code is derived from software contributed to The NetBSD Foundation
  8 * by Jason R. Thorpe and Andrew Doran.
  9 *
 10 * Redistribution and use in source and binary forms, with or without
 11 * modification, are permitted provided that the following conditions
 12 * are met:
 13 * 1. Redistributions of source code must retain the above copyright
 14 *    notice, this list of conditions and the following disclaimer.
 15 * 2. Redistributions in binary form must reproduce the above copyright
 16 *    notice, this list of conditions and the following disclaimer in the
 17 *    documentation and/or other materials provided with the distribution.
 18 *
 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 29 * POSSIBILITY OF SUCH DAMAGE.
 30 */
 31
 32/*
 33 * Machine-dependent spin lock operations.
 34 */
 35
 36#ifndef _X86_LOCK_H_
 37#define	_X86_LOCK_H_
 38
 39#include <sys/param.h>
 40
 41static __inline int
 42__SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr)
 43{
 44	return *__ptr == __SIMPLELOCK_LOCKED;
 45}
 46
 47static __inline int
 48__SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr)
 49{
 50	return *__ptr == __SIMPLELOCK_UNLOCKED;
 51}
 52
 53static __inline void
 54__cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
 55{
 56
 57	*__ptr = __SIMPLELOCK_LOCKED;
 58}
 59
 60static __inline void
 61__cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
 62{
 63
 64	*__ptr = __SIMPLELOCK_UNLOCKED;
 65}
 66
 67#ifdef _HARDKERNEL
 68# include <machine/cpufunc.h>
 69# define SPINLOCK_SPIN_HOOK	/* nothing */
 70# ifdef SPINLOCK_BACKOFF_HOOK
 71#  undef SPINLOCK_BACKOFF_HOOK
 72# endif
 73# define SPINLOCK_BACKOFF_HOOK	x86_pause()
 74# define SPINLOCK_INLINE
 75#else /* !_HARDKERNEL */
 76# define SPINLOCK_BODY
 77# define SPINLOCK_INLINE static __inline __unused
 78#endif /* _HARDKERNEL */
 79
 80SPINLOCK_INLINE void	__cpu_simple_lock_init(__cpu_simple_lock_t *);
 81SPINLOCK_INLINE void	__cpu_simple_lock(__cpu_simple_lock_t *);
 82SPINLOCK_INLINE int	__cpu_simple_lock_try(__cpu_simple_lock_t *);
 83SPINLOCK_INLINE void	__cpu_simple_unlock(__cpu_simple_lock_t *);
 84
 85#ifdef SPINLOCK_BODY
 86SPINLOCK_INLINE void
 87__cpu_simple_lock_init(__cpu_simple_lock_t *lockp)
 88{
 89
 90	*lockp = __SIMPLELOCK_UNLOCKED;
 91}
 92
 93SPINLOCK_INLINE int
 94__cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
 95{
 96	uint8_t val;
 97
 98	val = __SIMPLELOCK_LOCKED;
 99	__asm volatile ("xchgb %0,(%2)" : 
100	    "=qQ" (val)
101	    :"0" (val), "r" (lockp));
102	__insn_barrier();
103	return val == __SIMPLELOCK_UNLOCKED;
104}
105
106SPINLOCK_INLINE void
107__cpu_simple_lock(__cpu_simple_lock_t *lockp)
108{
109
110	while (!__cpu_simple_lock_try(lockp))
111		/* nothing */;
112	__insn_barrier();
113}
114
115/*
116 * Note on x86 memory ordering
117 *
118 * When releasing a lock we must ensure that no stores or loads from within
119 * the critical section are re-ordered by the CPU to occur outside of it:
120 * they must have completed and be visible to other processors once the lock
121 * has been released.
122 *
123 * NetBSD usually runs with the kernel mapped (via MTRR) in a WB (write
124 * back) memory region.  In that case, memory ordering on x86 platforms
125 * looks like this:
126 *
127 * i386		All loads/stores occur in instruction sequence.
128 *
129 * i486		All loads/stores occur in instruction sequence.  In
130 * Pentium	exceptional circumstances, loads can be re-ordered around
131 *		stores, but for the purposes of releasing a lock it does
132 *		not matter.  Stores may not be immediately visible to other
133 *		processors as they can be buffered.  However, since the
134 *		stores are buffered in order the lock release will always be
135 *		the last operation in the critical section that becomes
136 *		visible to other CPUs.
137 *
138 * Pentium Pro	The "Intel 64 and IA-32 Architectures Software Developer's
139 * onwards	Manual" volume 3A (order number 248966) says that (1) "Reads
140 *		can be carried out speculatively and in any order" and (2)
141 *		"Reads can pass buffered stores, but the processor is
142 *		self-consistent.".  This would be a problem for the below,
143 *		and would mandate a locked instruction cycle or load fence
144 *		before releasing the simple lock.
145 *
146 *		The "Intel Pentium 4 Processor Optimization" guide (order
147 *		number 253668-022US) says: "Loads can be moved before stores
148 *		that occurred earlier in the program if they are not
149 *		predicted to load from the same linear address.".  This is
150 *		not a problem since the only loads that can be re-ordered
151 *		take place once the lock has been released via a store.
152 *
153 *		The above two documents seem to contradict each other,
154 *		however with the exception of early steppings of the Pentium
155 *		Pro, the second document is closer to the truth: a store
156 *		will always act as a load fence for all loads that precede
157 *		the store in instruction order.
158 *
159 *		Again, note that stores can be buffered and will not always
160 *		become immediately visible to other CPUs: they are however
161 *		buffered in order.
162 *
163 * AMD64	Stores occur in order and are buffered.  Loads can be
164 *		reordered, however stores act as load fences, meaning that
165 *		loads can not be reordered around stores.
166 */
167SPINLOCK_INLINE void
168__cpu_simple_unlock(__cpu_simple_lock_t *lockp)
169{
170
171	__insn_barrier();
172	*lockp = __SIMPLELOCK_UNLOCKED;
173}
174
175#endif	/* SPINLOCK_BODY */
176
177#endif /* _X86_LOCK_H_ */