diff --git a/sys/compat/linuxkpi/common/include/linux/lockdep.h b/sys/compat/linuxkpi/common/include/linux/lockdep.h --- a/sys/compat/linuxkpi/common/include/linux/lockdep.h +++ b/sys/compat/linuxkpi/common/include/linux/lockdep.h @@ -70,17 +70,6 @@ #define lockdep_assert_none_held_once() do { } while (0) -static __inline bool -lockdep_is_held(void *__m) -{ - struct lock_object *__lock; - struct thread *__td; - - __lock = __m; - return (LOCK_CLASS(__lock)->lc_owner(__lock, &__td) != 0); -} -#define lockdep_is_held_type(_m, _t) lockdep_is_held(_m) - #else #define lockdep_assert(cond) do { } while (0) #define lockdep_assert_once(cond) do { } while (0) @@ -91,10 +80,23 @@ #define lockdep_assert_held_once(m) do { (void)(m); } while (0) -#define lockdep_is_held(m) 1 -#define lockdep_is_held_type(_m, _t) 1 #endif +static __inline bool +lockdep_is_held(void *__m __diagused) +{ +#ifdef INVARIANTS + struct lock_object *__lock; + struct thread *__td; + + __lock = __m; + return (LOCK_CLASS(__lock)->lc_owner(__lock, &__td) != 0); +#else + return (true); +#endif +} +#define lockdep_is_held_type(_m, _t) lockdep_is_held(_m) + #define might_lock(m) do { } while (0) #define might_lock_read(m) do { } while (0) #define might_lock_nested(m, n) do { } while (0) diff --git a/sys/compat/linuxkpi/common/include/linux/rcupdate.h b/sys/compat/linuxkpi/common/include/linux/rcupdate.h --- a/sys/compat/linuxkpi/common/include/linux/rcupdate.h +++ b/sys/compat/linuxkpi/common/include/linux/rcupdate.h @@ -1,6 +1,10 @@ /*- * Copyright (c) 2016-2017 Mellanox Technologies, Ltd. * All rights reserved. + * Copyright (c) 2024 The FreeBSD Foundation + * + * Portions of this software were developed by Björn Zeeb + * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,11 +30,23 @@ #ifndef _LINUXKPI_LINUX_RCUPDATE_H_ #define _LINUXKPI_LINUX_RCUPDATE_H_ +#include + #include #include +#include #include +extern int linuxkpi_rcu_debug; +#define RCU_WARN_ONCE(c, ...) \ +({ \ + if (unlikely(linuxkpi_rcu_debug <= 0)) \ + 1; \ + else \ + WARN_ONCE((c), ##__VA_ARGS__); \ +}) + #define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ /* BSD specific defines */ @@ -61,6 +77,9 @@ linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ } while (0) +#define rcu_read_lock_locked(void) \ + linux_rcu_read_lock_locked(RCU_TYPE_REGULAR) + #define synchronize_rcu(void) do { \ linux_synchronize_rcu(RCU_TYPE_REGULAR); \ } while (0) @@ -79,14 +98,34 @@ #define rcu_access_pointer(p) \ ((__typeof(*p) *)READ_ONCE(p)) -#define rcu_dereference_protected(p, c) \ +#define rcu_dereference(p) \ ((__typeof(*p) *)READ_ONCE(p)) -#define rcu_dereference(p) \ - rcu_dereference_protected(p, 0) +#define __rcu_var_name(n, f, l) \ + __CONCAT(__CONCAT(__CONCAT(rcu_, n), _), __COUNTER__) + +#define __rcu_dereference_protected(p, c, n) \ +({ \ + RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ + __func__, __LINE__, __XSTRING(n)); \ + rcu_dereference(p); \ +}) + +#define rcu_dereference_protected(p, c) \ + __rcu_dereference_protected((p), (c), \ + __rcu_var_name(protected, __func__, __LINE__)) + +#define __rcu_dereference_check(p, c, n) \ +({ \ + __typeof(*p) *n = rcu_dereference(p); \ + RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ + __func__, __LINE__, __XSTRING(n)); \ + n; \ +}) -#define rcu_dereference_check(p, c) \ - rcu_dereference_protected(p, c) +#define rcu_dereference_check(p, c) \ + __rcu_dereference_check((p), (c) || rcu_read_lock_locked(), \ + __rcu_var_name(check, __func__, __LINE__)) #define rcu_dereference_raw(p) \ ((__typeof(*p) *)READ_ONCE(p)) @@ -113,11 +152,12 @@ /* prototypes */ -extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); -extern void linux_rcu_barrier(unsigned type); -extern void linux_rcu_read_lock(unsigned type); -extern void linux_rcu_read_unlock(unsigned type); -extern void linux_synchronize_rcu(unsigned type); +void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); +void linux_rcu_barrier(unsigned type); +void linux_rcu_read_lock(unsigned type); +void linux_rcu_read_unlock(unsigned type); +bool linux_rcu_read_lock_locked(unsigned); +void linux_synchronize_rcu(unsigned type); /* Empty implementation for !DEBUG */ #define init_rcu_head(...) diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c --- a/sys/compat/linuxkpi/common/src/linux_compat.c +++ b/sys/compat/linuxkpi/common/src/linux_compat.c @@ -117,6 +117,10 @@ SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug, CTLFLAG_RWTUN, &linuxkpi_debug, 0, "Set to enable pr_debug() prints. Clear to disable."); +int linuxkpi_rcu_debug; +SYSCTL_INT(_compat_linuxkpi, OID_AUTO, rcu_debug, CTLFLAG_RWTUN, + &linuxkpi_rcu_debug, 0, "Set to enable RCU warning. Clear to disable."); + int linuxkpi_warn_dump_stack = 0; SYSCTL_INT(_compat_linuxkpi, OID_AUTO, warn_dump_stack, CTLFLAG_RWTUN, &linuxkpi_warn_dump_stack, 0, diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c b/sys/compat/linuxkpi/common/src/linux_rcu.c --- a/sys/compat/linuxkpi/common/src/linux_rcu.c +++ b/sys/compat/linuxkpi/common/src/linux_rcu.c @@ -2,6 +2,10 @@ * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) * Copyright (c) 2017-2021 Hans Petter Selasky (hselasky@freebsd.org) * All rights reserved. + * Copyright (c) 2024 The FreeBSD Foundation + * + * Portions of this software were developed by Björn Zeeb + * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -245,6 +249,30 @@ sched_unpin(); } +bool +linux_rcu_read_lock_locked(unsigned type) +{ + struct linux_epoch_record *record; + struct task_struct *ts; + bool active; + + MPASS(type < RCU_TYPE_MAX); + + if (RCU_SKIP()) + return (false); + + ts = current; + if (ts->rcu_recurse[type] == 0) + return (false); + + record = &DPCPU_GET(linux_epoch_record[type]); + critical_enter(); + active = record->epoch_record.active != 0; + critical_exit(); + + return (active); +} + static void linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, ck_epoch_record_t *epoch_record, void *arg __unused) {