[Midnightbsd-cvs] src [12365] trunk/sys/sys/rwlock.h: sync with FreeBSD 11-stable
laffer1 at midnightbsd.org
laffer1 at midnightbsd.org
Sun Feb 9 13:34:37 EST 2020
Revision: 12365
http://svnweb.midnightbsd.org/src/?rev=12365
Author: laffer1
Date: 2020-02-09 13:34:36 -0500 (Sun, 09 Feb 2020)
Log Message:
-----------
sync with FreeBSD 11-stable
Modified Paths:
--------------
trunk/sys/sys/rwlock.h
Modified: trunk/sys/sys/rwlock.h
===================================================================
--- trunk/sys/sys/rwlock.h 2020-02-09 18:34:16 UTC (rev 12364)
+++ trunk/sys/sys/rwlock.h 2020-02-09 18:34:36 UTC (rev 12365)
@@ -24,7 +24,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * $FreeBSD: stable/10/sys/sys/rwlock.h 323870 2017-09-21 19:24:11Z marius $
+ * $FreeBSD: stable/11/sys/sys/rwlock.h 343420 2019-01-25 11:01:11Z kib $
*/
#ifndef _SYS_RWLOCK_H_
@@ -59,13 +59,14 @@
#define RW_LOCK_READ_WAITERS 0x02
#define RW_LOCK_WRITE_WAITERS 0x04
#define RW_LOCK_WRITE_SPINNER 0x08
+#define RW_LOCK_WRITER_RECURSED 0x10
#define RW_LOCK_FLAGMASK \
(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS | \
- RW_LOCK_WRITE_SPINNER)
+ RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED)
#define RW_LOCK_WAITERS (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)
#define RW_OWNER(x) ((x) & ~RW_LOCK_FLAGMASK)
-#define RW_READERS_SHIFT 4
+#define RW_READERS_SHIFT 5
#define RW_READERS(x) (RW_OWNER((x)) >> RW_READERS_SHIFT)
#define RW_READERS_LOCK(x) ((x) << RW_READERS_SHIFT | RW_LOCK_READ)
#define RW_ONE_READER (1 << RW_READERS_SHIFT)
@@ -77,6 +78,8 @@
#define rw_recurse lock_object.lo_data
+#define RW_READ_VALUE(x) ((x)->rw_lock)
+
/* Very simple operations on rw_lock. */
/* Try to obtain a write lock once. */
@@ -83,10 +86,16 @@
#define _rw_write_lock(rw, tid) \
atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
+#define _rw_write_lock_fetch(rw, vp, tid) \
+ atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
+
/* Release a write lock quickly if there are no waiters. */
#define _rw_write_unlock(rw, tid) \
atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
+#define _rw_write_unlock_fetch(rw, tid) \
+ atomic_fcmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
+
/*
* Full lock operations that are suitable to be inlined in non-debug
* kernels. If the lock cannot be acquired or released trivially then
@@ -96,22 +105,20 @@
/* Acquire a write lock. */
#define __rw_wlock(rw, tid, file, line) do { \
uintptr_t _tid = (uintptr_t)(tid); \
- \
- if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\
- _rw_wlock_hard((rw), _tid, (file), (line)); \
- else \
- LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_RW_WLOCK_ACQUIRE, \
- rw, 0, 0, (file), (line)); \
+ uintptr_t _v = RW_UNLOCKED; \
+ \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) || \
+ !_rw_write_lock_fetch((rw), &_v, _tid))) \
+ _rw_wlock_hard((rw), _v, (file), (line)); \
} while (0)
/* Release a write lock. */
#define __rw_wunlock(rw, tid, file, line) do { \
- uintptr_t _tid = (uintptr_t)(tid); \
+ uintptr_t _v = (uintptr_t)(tid); \
\
- if ((rw)->rw_recurse) \
- (rw)->rw_recurse--; \
- else if (!_rw_write_unlock((rw), _tid)) \
- _rw_wunlock_hard((rw), _tid, (file), (line)); \
+ if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) || \
+ !_rw_write_unlock_fetch((rw), &_v))) \
+ _rw_wunlock_hard((rw), _v, (file), (line)); \
} while (0)
/*
@@ -122,19 +129,24 @@
void _rw_init_flags(volatile uintptr_t *c, const char *name, int opts);
void _rw_destroy(volatile uintptr_t *c);
void rw_sysinit(void *arg);
-void rw_sysinit_flags(void *arg);
int _rw_wowned(const volatile uintptr_t *c);
void _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_wlock(volatile uintptr_t *c, const char *file, int line);
void _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line);
+void __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_rlock(volatile uintptr_t *c, const char *file, int line);
+int __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_rlock(volatile uintptr_t *c, const char *file, int line);
+void _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line);
-void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file,
- int line);
-void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid,
- const char *file, int line);
+void __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+void __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t v
+ LOCK_FILE_LINE_ARG_DEF);
+int __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
int __rw_try_upgrade(volatile uintptr_t *c, const char *file, int line);
+void __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF);
void __rw_downgrade(volatile uintptr_t *c, const char *file, int line);
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
void __rw_assert(const volatile uintptr_t *c, int what, const char *file,
@@ -160,20 +172,38 @@
__rw_try_wlock(&(rw)->rw_lock, f, l)
#define _rw_wunlock(rw, f, l) \
_rw_wunlock_cookie(&(rw)->rw_lock, f, l)
+#define _rw_try_rlock(rw, f, l) \
+ __rw_try_rlock(&(rw)->rw_lock, f, l)
+#if LOCK_DEBUG > 0
#define _rw_rlock(rw, f, l) \
__rw_rlock(&(rw)->rw_lock, f, l)
-#define _rw_try_rlock(rw, f, l) \
- __rw_try_rlock(&(rw)->rw_lock, f, l)
#define _rw_runlock(rw, f, l) \
_rw_runlock_cookie(&(rw)->rw_lock, f, l)
-#define _rw_wlock_hard(rw, t, f, l) \
- __rw_wlock_hard(&(rw)->rw_lock, t, f, l)
-#define _rw_wunlock_hard(rw, t, f, l) \
- __rw_wunlock_hard(&(rw)->rw_lock, t, f, l)
+#else
+#define _rw_rlock(rw, f, l) \
+ __rw_rlock_int((struct rwlock *)rw)
+#define _rw_runlock(rw, f, l) \
+ _rw_runlock_cookie_int((struct rwlock *)rw)
+#endif
+#if LOCK_DEBUG > 0
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v, f, l)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v, f, l)
#define _rw_try_upgrade(rw, f, l) \
__rw_try_upgrade(&(rw)->rw_lock, f, l)
#define _rw_downgrade(rw, f, l) \
__rw_downgrade(&(rw)->rw_lock, f, l)
+#else
+#define _rw_wlock_hard(rw, v, f, l) \
+ __rw_wlock_hard(&(rw)->rw_lock, v)
+#define _rw_wunlock_hard(rw, v, f, l) \
+ __rw_wunlock_hard(&(rw)->rw_lock, v)
+#define _rw_try_upgrade(rw, f, l) \
+ __rw_try_upgrade_int(rw)
+#define _rw_downgrade(rw, f, l) \
+ __rw_downgrade_int(rw)
+#endif
#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
#define _rw_assert(rw, w, f, l) \
__rw_assert(&(rw)->rw_lock, w, f, l)
@@ -212,23 +242,19 @@
_sleep((chan), &(rw)->lock_object, (pri), (wmesg), \
tick_sbt * (timo), 0, C_HARDCLOCK)
-#define rw_initialized(rw) lock_initalized(&(rw)->lock_object)
+#define rw_initialized(rw) lock_initialized(&(rw)->lock_object)
struct rw_args {
void *ra_rw;
const char *ra_desc;
-};
-
-struct rw_args_flags {
- void *ra_rw;
- const char *ra_desc;
int ra_flags;
};
-#define RW_SYSINIT(name, rw, desc) \
+#define RW_SYSINIT_FLAGS(name, rw, desc, flags) \
static struct rw_args name##_args = { \
(rw), \
(desc), \
+ (flags), \
}; \
SYSINIT(name##_rw_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
rw_sysinit, &name##_args); \
@@ -235,18 +261,8 @@
SYSUNINIT(name##_rw_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
_rw_destroy, __DEVOLATILE(void *, &(rw)->rw_lock))
+#define RW_SYSINIT(name, rw, desc) RW_SYSINIT_FLAGS(name, rw, desc, 0)
-#define RW_SYSINIT_FLAGS(name, rw, desc, flags) \
- static struct rw_args_flags name##_args = { \
- (rw), \
- (desc), \
- (flags), \
- }; \
- SYSINIT(name##_rw_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
- rw_sysinit_flags, &name##_args); \
- SYSUNINIT(name##_rw_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \
- _rw_destroy, __DEVOLATILE(void *, &(rw)->rw_lock))
-
/*
* Options passed to rw_init_flags().
*/
More information about the Midnightbsd-cvs
mailing list