Page MenuHomeFreeBSD
Files F146318560

D45624.id139936.diff
No OneTemporary
Actions

D45624.id139936.diff
View Options

diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -184,6 +184,9 @@
int flags, bool fp);
static bool __always_inline lockmgr_sunlock_try(struct lock *lk, uintptr_t *xp);
+static bool __always_inline lockmgr_sleepgen_valid(struct lock *lk,
+ u_int sleepgen);
+
static void
lockmgr_exit(u_int flags, struct lock_object *ilk, int wakeup_swapper)
{
@@ -581,7 +584,7 @@
static __noinline int
lockmgr_slock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
- const char *file, int line, struct lockmgr_wait *lwa)
+ u_int sleepgen, const char *file, int line, struct lockmgr_wait *lwa)
{
uintptr_t tid, x;
int error = 0;
@@ -653,6 +656,14 @@
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
+
+ /* Check sleepgen. */
+ if (!lockmgr_sleepgen_valid(lk, sleepgen)) {
+ sleepq_release(&lk->lock_object);
+ error = ENOLCK;
+ break;
+ }
+
x = lockmgr_read_value(lk);
retry_sleepq:
@@ -766,7 +777,7 @@
static __noinline int
lockmgr_xlock_hard(struct lock *lk, u_int flags, struct lock_object *ilk,
- const char *file, int line, struct lockmgr_wait *lwa)
+ u_int sleepgen, const char *file, int line, struct lockmgr_wait *lwa)
{
struct lock_class *class;
uintptr_t tid, x, v;
@@ -868,6 +879,14 @@
* probabilly will need to manipulate waiters flags.
*/
sleepq_lock(&lk->lock_object);
+
+ /* Check sleepgen. */
+ if (!lockmgr_sleepgen_valid(lk, sleepgen)) {
+ sleepq_release(&lk->lock_object);
+ error = ENOLCK;
+ break;
+ }
+
x = lockmgr_read_value(lk);
retry_sleepq:
@@ -1024,7 +1043,8 @@
}
out_xlock:
- error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
+ error = lockmgr_xlock_hard(lk, flags, ilk, LK_SLEEPGEN_INVALID, file,
+ line, lwa);
flags &= ~LK_INTERLOCK;
out:
lockmgr_exit(flags, ilk, 0);
@@ -1043,6 +1063,8 @@
if (SCHEDULER_STOPPED())
return (0);
+ /* XXX SLEEPGEN ASSERTS? */
+
op = flags & LK_TYPE_MASK;
locked = false;
switch (op) {
@@ -1058,8 +1080,8 @@
file, line, flags);
locked = true;
} else {
- return (lockmgr_slock_hard(lk, flags, ilk, file, line,
- NULL));
+ return (lockmgr_slock_hard(lk, flags, ilk,
+ LK_SLEEPGEN_INVALID, file, line, NULL));
}
break;
case LK_EXCLUSIVE:
@@ -1074,8 +1096,8 @@
flags);
locked = true;
} else {
- return (lockmgr_xlock_hard(lk, flags, ilk, file, line,
- NULL));
+ return (lockmgr_xlock_hard(lk, flags, ilk,
+ LK_SLEEPGEN_INVALID, file, line, NULL));
}
break;
case LK_UPGRADE:
@@ -1243,7 +1265,8 @@
return (0);
}
- return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
+ return (lockmgr_slock_hard(lk, flags | LK_ADAPTIVE, NULL,
+ LK_SLEEPGEN_INVALID, file, line, NULL));
}
int
@@ -1264,7 +1287,8 @@
return (0);
}
- return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL, file, line, NULL));
+ return (lockmgr_xlock_hard(lk, flags | LK_ADAPTIVE, NULL,
+ LK_SLEEPGEN_INVALID, file, line, NULL));
}
int
@@ -1299,15 +1323,16 @@
}
int
-__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
+__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk_,
const char *wmesg, int pri, int timo, const char *file, int line)
{
GIANT_DECLARE;
struct lockmgr_wait lwa;
struct lock_class *class;
+ struct lock_object *ilk;
const char *iwmesg;
uintptr_t tid, v, x;
- u_int op, realexslp;
+ u_int op, realexslp, sleepgen;
int error, ipri, itimo, queue, wakeup_swapper;
#ifdef LOCK_PROFILING
uint64_t waittime = 0;
@@ -1324,6 +1349,14 @@
ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
+ /* XXX may want to push some of this down into slock and xlock hard */
+ ilk = NULL;
+ sleepgen = LK_SLEEPGEN_INVALID;
+ if ((flags & LK_INTERLOCK) != 0)
+ ilk = ilk_;
+ else if ((flags & LK_SLEEPGEN) != 0)
+ sleepgen = (uintptr_t)ilk_;
+
lwa.iwmesg = iwmesg;
lwa.ipri = ipri;
lwa.itimo = itimo;
@@ -1338,6 +1371,15 @@
KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
__func__, file, line));
+ KASSERT((flags & (LK_INTERLOCK | LK_SLEEPGEN)) !=
+ (LK_INTERLOCK | LK_SLEEPGEN),
+ ("%s: Invalid flags (LK_INTERLOCK | LK_SLEEPGEN) @ %s:%d ",
+ __func__, file, line));
+ KASSERT((flags & LK_SLEEPGEN) == 0 || (sleepgen & 1) == 1,
+ ("%s: Invalid sleepgen @ %s:%d", __func__, file, line));
+ KASSERT((flags & LK_SLEEPGEN) == 0 || op == LK_SHARED ||
+ op == LK_EXCLUSIVE,
+ ("%s: Invalid op with LK_SLEEPGEN @ %s:%d", __func__, file, line));
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("%s: idle thread %p on lockmgr %s @ %s:%d", __func__, curthread,
lk->lock_object.lo_name, file, line));
@@ -1363,14 +1405,16 @@
wakeup_swapper = 0;
switch (op) {
case LK_SHARED:
- return (lockmgr_slock_hard(lk, flags, ilk, file, line, &lwa));
+ return (lockmgr_slock_hard(lk, flags, ilk, sleepgen, file,
+ line, &lwa));
break;
case LK_UPGRADE:
case LK_TRYUPGRADE:
return (lockmgr_upgrade(lk, flags, ilk, file, line, &lwa));
break;
case LK_EXCLUSIVE:
- return (lockmgr_xlock_hard(lk, flags, ilk, file, line, &lwa));
+ return (lockmgr_xlock_hard(lk, flags, ilk, sleepgen, file,
+ line, &lwa));
break;
case LK_DOWNGRADE:
_lockmgr_assert(lk, KA_XLOCKED, file, line);
@@ -1653,6 +1697,71 @@
}
}
+static __always_inline bool
+lockmgr_sleepgen_valid(struct lock *lk, u_int sleepgen)
+{
+ return (sleepgen == LK_SLEEPGEN_INVALID ||
+ atomic_load_int(&((struct lock_sleepgen *)lk)->lksg_sleepgen) ==
+ sleepgen);
+}
+
+static void
+lockmgr_wakeall(struct lock *lk)
+{
+ uintptr_t x, v;
+ int wakeup_swapper;
+
+ wakeup_swapper = 0;
+
+ sleepq_lock(&lk->lock_object);
+ lk->lk_exslpfail = 0;
+ for (;;) {
+ x = lockmgr_read_value(lk);
+ v = x & ~LK_ALL_WAITERS;
+ if (x == v || atomic_fcmpset_ptr(&lk->lk_lock, &x, v))
+ break;
+ cpu_spinwait();
+ }
+ if ((x & LK_EXCLUSIVE_WAITERS) != 0) {
+ wakeup_swapper |= sleepq_broadcast(&lk->lock_object,
+ SLEEPQ_LK, 0, SQ_EXCLUSIVE_QUEUE);
+ }
+ if ((x & LK_SHARED_WAITERS) != 0) {
+ wakeup_swapper |= sleepq_broadcast(&lk->lock_object,
+ SLEEPQ_LK, 0, SQ_SHARED_QUEUE);
+ }
+ sleepq_release(&lk->lock_object);
+
+ if (wakeup_swapper)
+ kick_proc0();
+}
+
+void
+_lockmgr_sleepgen_invalidate_hard(struct lock_sleepgen *lksg)
+{
+ atomic_add_int(&lksg->lksg_sleepgen, LK_SLEEPGEN_INCR);
+ lockmgr_wakeall(&lksg->lksg_lock);
+}
+
+u_int (
+lockmgr_sleepgen_acquire)(struct lock_sleepgen *lksg)
+{
+ return _lockmgr_sleepgen_acquire(lksg);
+}
+
+void (
+lockmgr_sleepgen_release)(struct lock_sleepgen *lksg)
+{
+ _lockmgr_sleepgen_release(lksg);
+}
+
+
+void (
+lockmgr_sleepgen_invalidate)(struct lock_sleepgen *lksg)
+{
+ _lockmgr_sleepgen_invalidate(lksg);
+}
+
void
lockmgr_printinfo(const struct lock *lk)
{
diff --git a/sys/sys/_lockmgr.h b/sys/sys/_lockmgr.h
--- a/sys/sys/_lockmgr.h
+++ b/sys/sys/_lockmgr.h
@@ -46,4 +46,10 @@
#endif
};
+struct lock_sleepgen {
+ struct lock lksg_lock;
+ u_int lksg_holders;
+ u_int lksg_sleepgen;
+};
+
#endif
diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h
--- a/sys/sys/lockmgr.h
+++ b/sys/sys/lockmgr.h
@@ -115,6 +115,15 @@
NULL, wmesg, prio, timo, file, line));
}
+static __inline int
+_lockmgr_args_sleepgen(struct lock_sleepgen *lksg, u_int flags, u_int sleepgen,
+ const char *wmesg, int prio, int timo, const char *file, int line)
+{
+
+ return (__lockmgr_args(&lksg->lksg_lock, flags,
+ (void *)(uintptr_t)sleepgen, wmesg, prio, timo, file, line));
+}
+
/*
* Define aliases in order to complete lockmgr KPI.
*/
@@ -173,6 +182,7 @@
#define LK_TIMELOCK 0x001000
#define LK_NODDLKTREAT 0x002000
#define LK_ADAPTIVE 0x004000
+#define LK_SLEEPGEN 0x008000
/*
* Operations for lockmgr().
@@ -208,6 +218,61 @@
#define KA_NOTRECURSED LA_NOTRECURSED
#endif
+/* XXX organize me */
+#define LK_SLEEPGEN_INVALID 0
+#define LK_SLEEPGEN_INIT 1
+#define LK_SLEEPGEN_INCR 2
+
+#define _lockmgr_sleepgen_acquire(lksg) __extension__ ({ \
+ atomic_add_acq_int(&(lksg)->lksg_holders, 1); \
+ atomic_load_acq_int(&(lksg)->lksg_sleepgen); \
+})
+
+#define _lockmgr_sleepgen_release(lksg) \
+ atomic_add_int(&(lksg)->lksg_holders, -1)
+
+#define _lockmgr_sleepgen_invalidate(lksg) do { \
+ lockmgr_assert(&(lksg)->lksg_lock, KA_LOCKED); \
+ KASSERT(((lksg)->lksg_sleepgen & 1) == 1, \
+ ("lockmgr_sleepgen_invalidate: lock %p bad sleepgen %u", \
+ (lksg), (lksg)->lksg_sleepgen)); \
+ atomic_thread_fence_rel(); \
+ if (atomic_load_int(&(lksg)->lksg_holders) != 0) \
+ _lockmgr_sleepgen_invalidate_hard((lksg)); \
+} while (0)
+
+u_int lockmgr_sleepgen_acquire(struct lock_sleepgen *);
+void lockmgr_sleepgen_release(struct lock_sleepgen *);
+void lockmgr_sleepgen_invalidate(struct lock_sleepgen *);
+void _lockmgr_sleepgen_invalidate_hard(struct lock_sleepgen *);
+
+/* Macros for inline expansion. */
+#define lockmgr_sleepgen_acquire(lksg) _lockmgr_sleepgen_acquire((lksg))
+#define lockmgr_sleepgen_release(lksg) _lockmgr_sleepgen_release((lksg))
+#define lockmgr_sleepgen_invalidate(lksg) \
+ _lockmgr_sleepgen_invalidate((lksg))
+
+#define lockmgr_args_sleepgen_cond(lksg, flags, wmesg, prio, timo, \
+ cond) __extension__ ({ \
+ int _error; \
+ u_int _sleepgen; \
+ _sleepgen = lockmgr_sleepgen_acquire((lksg)); \
+ /* Check cond after acquiring the sleepgen. */ \
+ if (cond) { \
+ _error = _lockmgr_args_sleepgen((lksg), (flags), \
+ _sleepgen, (wmesg), (prio), (timo), LOCK_FILE, \
+ LOCK_LINE); \
+ } else { \
+ _error = ENOLCK; \
+ } \
+ lockmgr_sleepgen_release((lksg)); \
+ _error; \
+})
+
+#define lockmgr_sleepgen_cond(lksg, flags, wmesg, cond) \
+ lockmgr_args_sleepgen_cond((lksg), (flags), (wmesg), \
+ LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, (cond))
+
#endif /* _KERNEL */
#endif /* !_SYS_LOCKMGR_H_ */

File Metadata

Mime Type
text/plain
Expires
Mon, Mar 2, 5:07 PM (14 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
29142138
Default Alt Text
D45624.id139936.diff (9 KB)

Event Timeline