1/*-
  2 * Copyright (c) 2014 Mateusz Guzik <mjg@FreeBSD.org>
  3 *
  4 * Redistribution and use in source and binary forms, with or without
  5 * modification, are permitted provided that the following conditions
  6 * are met:
  7 * 1. Redistributions of source code must retain the above copyright
  8 *    notice, this list of conditions and the following disclaimer.
  9 * 2. Redistributions in binary form must reproduce the above copyright
 10 *    notice, this list of conditions and the following disclaimer in the
 11 *    documentation and/or other materials provided with the distribution.
 12 *
 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 23 * SUCH DAMAGE.
 24 */
 25
 26#ifndef _SYS_SEQC_H_
 27#define _SYS_SEQC_H_
 28
 29#ifdef _KERNEL
 30#include <sys/systm.h>
 31#endif
 32#include <sys/types.h>
 33
 34/*
 35 * seqc_t may be included in structs visible to userspace
 36 */
 37#include <sys/_seqc.h>
 38
 39#ifdef _KERNEL
 40
 41/* A hack to get MPASS macro */
 42#include <sys/lock.h>
 43
 44#include <machine/cpu.h>
 45
 46#define	SEQC_MOD	1
 47
 48/*
 49 * Predicts from inline functions are not honored by clang.
 50 */
 51#define seqc_in_modify(seqc)	({			\
 52	seqc_t __seqc = (seqc);				\
 53							\
 54	__predict_false(__seqc & SEQC_MOD);		\
 55})
 56
 57static __inline void
 58seqc_write_begin(seqc_t *seqcp)
 59{
 60
 61	critical_enter();
 62	MPASS(!seqc_in_modify(*seqcp));
 63	*seqcp += SEQC_MOD;
 64	atomic_thread_fence_rel();
 65}
 66
 67static __inline void
 68seqc_write_end(seqc_t *seqcp)
 69{
 70
 71	atomic_thread_fence_rel();
 72	*seqcp += SEQC_MOD;
 73	MPASS(!seqc_in_modify(*seqcp));
 74	critical_exit();
 75}
 76
 77static __inline seqc_t
 78seqc_read_any(const seqc_t *seqcp)
 79{
 80
 81	return (atomic_load_acq_int(__DECONST(seqc_t *, seqcp)));
 82}
 83
 84static __inline seqc_t
 85seqc_read_notmodify(const seqc_t *seqcp)
 86{
 87
 88	return (atomic_load_acq_int(__DECONST(seqc_t *, seqcp)) & ~SEQC_MOD);
 89}
 90
 91static __inline seqc_t
 92seqc_read(const seqc_t *seqcp)
 93{
 94	seqc_t ret;
 95
 96	for (;;) {
 97		ret = seqc_read_any(seqcp);
 98		if (seqc_in_modify(ret)) {
 99			cpu_spinwait();
100			continue;
101		}
102		break;
103	}
104
105	return (ret);
106}
107
108#define seqc_consistent_no_fence(seqcp, oldseqc)({	\
109	const seqc_t *__seqcp = (seqcp);		\
110	seqc_t __oldseqc = (oldseqc);			\
111							\
112	MPASS(!(seqc_in_modify(__oldseqc)));		\
113	__predict_true(*__seqcp == __oldseqc);		\
114})
115
116#define seqc_consistent(seqcp, oldseqc)		({	\
117	atomic_thread_fence_acq();			\
118	seqc_consistent_no_fence(seqcp, oldseqc);	\
119})
120
121/*
122 * Variant which does not critical enter/exit.
123 */
124static __inline void
125seqc_sleepable_write_begin(seqc_t *seqcp)
126{
127
128	MPASS(!seqc_in_modify(*seqcp));
129	*seqcp += SEQC_MOD;
130	atomic_thread_fence_rel();
131}
132
133static __inline void
134seqc_sleepable_write_end(seqc_t *seqcp)
135{
136
137	atomic_thread_fence_rel();
138	*seqcp += SEQC_MOD;
139	MPASS(!seqc_in_modify(*seqcp));
140}
141
142#endif	/* _KERNEL */
143#endif	/* _SYS_SEQC_H_ */