Changeset View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 478 Lines • ▼ Show 20 Lines | |||||
static struct lock_object invl_gen_ts = { | static struct lock_object invl_gen_ts = { | ||||
.lo_name = "invlts", | .lo_name = "invlts", | ||||
}; | }; | ||||
static struct pmap_invl_gen pmap_invl_gen_head = { | static struct pmap_invl_gen pmap_invl_gen_head = { | ||||
.gen = 1, | .gen = 1, | ||||
.next = NULL, | .next = NULL, | ||||
}; | }; | ||||
static u_long pmap_invl_gen = 1; | static u_long pmap_invl_gen = 1; | ||||
static int pmap_invl_waiters; | |||||
static struct callout pmap_invl_callout; | |||||
static bool pmap_invl_callout_inited; | |||||
#define PMAP_ASSERT_NOT_IN_DI() \ | #define PMAP_ASSERT_NOT_IN_DI() \ | ||||
KASSERT(pmap_not_in_di(), ("DI already started")) | KASSERT(pmap_not_in_di(), ("DI already started")) | ||||
static bool | static bool | ||||
pmap_di_locked(void) | pmap_di_locked(void) | ||||
{ | { | ||||
int tun; | int tun; | ||||
Show All 38 Lines | |||||
pmap_thread_init_invl_gen_l(struct thread *td) | pmap_thread_init_invl_gen_l(struct thread *td) | ||||
{ | { | ||||
struct pmap_invl_gen *invl_gen; | struct pmap_invl_gen *invl_gen; | ||||
invl_gen = &td->td_md.md_invl_gen; | invl_gen = &td->td_md.md_invl_gen; | ||||
invl_gen->gen = 0; | invl_gen->gen = 0; | ||||
} | } | ||||
static void | |||||
pmap_delayed_invl_wait_block(u_long *m_gen, u_long *invl_gen) | |||||
{ | |||||
struct turnstile *ts; | |||||
ts = turnstile_trywait(&invl_gen_ts); | |||||
if (*m_gen > atomic_load_long(invl_gen)) | |||||
turnstile_wait(ts, NULL, TS_SHARED_QUEUE); | |||||
else | |||||
turnstile_cancel(ts); | |||||
} | |||||
static void | |||||
pmap_delayed_invl_finish_unblock(u_long new_gen) | |||||
{ | |||||
struct turnstile *ts; | |||||
turnstile_chain_lock(&invl_gen_ts); | |||||
ts = turnstile_lookup(&invl_gen_ts); | |||||
if (new_gen != 0) | |||||
pmap_invl_gen = new_gen; | |||||
if (ts != NULL) { | |||||
turnstile_broadcast(ts, TS_SHARED_QUEUE); | |||||
turnstile_unpend(ts); | |||||
} | |||||
turnstile_chain_unlock(&invl_gen_ts); | |||||
} | |||||
/* | /* | ||||
* Start a new Delayed Invalidation (DI) block of code, executed by | * Start a new Delayed Invalidation (DI) block of code, executed by | ||||
* the current thread. Within a DI block, the current thread may | * the current thread. Within a DI block, the current thread may | ||||
* destroy both the page table and PV list entries for a mapping and | * destroy both the page table and PV list entries for a mapping and | ||||
* then release the corresponding PV list lock before ensuring that | * then release the corresponding PV list lock before ensuring that | ||||
* the mapping is flushed from the TLBs of any processors with the | * the mapping is flushed from the TLBs of any processors with the | ||||
* pmap active. | * pmap active. | ||||
*/ | */ | ||||
Show All 28 Lines | |||||
* earlier DI had finished. Instead, this function bumps the earlier | * earlier DI had finished. Instead, this function bumps the earlier | ||||
* DI's generation number to match the generation number of the | * DI's generation number to match the generation number of the | ||||
* current thread's DI. | * current thread's DI. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_delayed_invl_finish_l(void) | pmap_delayed_invl_finish_l(void) | ||||
{ | { | ||||
struct pmap_invl_gen *invl_gen, *next; | struct pmap_invl_gen *invl_gen, *next; | ||||
struct turnstile *ts; | |||||
invl_gen = &curthread->td_md.md_invl_gen; | invl_gen = &curthread->td_md.md_invl_gen; | ||||
KASSERT(invl_gen->gen != 0, ("missed invl_start")); | KASSERT(invl_gen->gen != 0, ("missed invl_start")); | ||||
mtx_lock(&invl_gen_mtx); | mtx_lock(&invl_gen_mtx); | ||||
next = LIST_NEXT(invl_gen, link); | next = LIST_NEXT(invl_gen, link); | ||||
if (next == NULL) { | if (next == NULL) | ||||
turnstile_chain_lock(&invl_gen_ts); | pmap_delayed_invl_finish_unblock(invl_gen->gen); | ||||
ts = turnstile_lookup(&invl_gen_ts); | else | ||||
pmap_invl_gen = invl_gen->gen; | |||||
if (ts != NULL) { | |||||
turnstile_broadcast(ts, TS_SHARED_QUEUE); | |||||
turnstile_unpend(ts); | |||||
} | |||||
turnstile_chain_unlock(&invl_gen_ts); | |||||
} else { | |||||
next->gen = invl_gen->gen; | next->gen = invl_gen->gen; | ||||
} | |||||
LIST_REMOVE(invl_gen, link); | LIST_REMOVE(invl_gen, link); | ||||
mtx_unlock(&invl_gen_mtx); | mtx_unlock(&invl_gen_mtx); | ||||
invl_gen->gen = 0; | invl_gen->gen = 0; | ||||
} | } | ||||
static bool | static bool | ||||
pmap_not_in_di_u(void) | pmap_not_in_di_u(void) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 240 Lines • ▼ Show 20 Lines | if (!pmap_delayed_invl_finish_u_crit(invl_gen, p)) { | ||||
atomic_clear_ptr((uintptr_t *)&invl_gen->next, | atomic_clear_ptr((uintptr_t *)&invl_gen->next, | ||||
PMAP_INVL_GEN_NEXT_INVALID); | PMAP_INVL_GEN_NEXT_INVALID); | ||||
critical_exit(); | critical_exit(); | ||||
PV_STAT(atomic_add_long(&invl_finish_restart, 1)); | PV_STAT(atomic_add_long(&invl_finish_restart, 1)); | ||||
lock_delay(&lda); | lock_delay(&lda); | ||||
goto again; | goto again; | ||||
} | } | ||||
critical_exit(); | critical_exit(); | ||||
if (atomic_load_int(&pmap_invl_waiters) > 0) | |||||
pmap_delayed_invl_finish_unblock(0); | |||||
if (invl_gen->saved_pri != 0) { | if (invl_gen->saved_pri != 0) { | ||||
thread_lock(td); | thread_lock(td); | ||||
sched_prio(td, invl_gen->saved_pri); | sched_prio(td, invl_gen->saved_pri); | ||||
thread_unlock(td); | thread_unlock(td); | ||||
} | } | ||||
} | } | ||||
#ifdef DDB | #ifdef DDB | ||||
Show All 16 Lines | DB_SHOW_COMMAND(di_queue, pmap_di_queue) | ||||
} | } | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
static long invl_wait; | static long invl_wait; | ||||
SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0, | SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0, | ||||
"Number of times DI invalidation blocked pmap_remove_all/write"); | "Number of times DI invalidation blocked pmap_remove_all/write"); | ||||
static long invl_wait_slow; | |||||
SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, &invl_wait_slow, 0, | |||||
"Number of slow invalidation waits for lockless DI"); | |||||
#endif | #endif | ||||
static u_long * | static u_long * | ||||
pmap_delayed_invl_genp(vm_page_t m) | pmap_delayed_invl_genp(vm_page_t m) | ||||
{ | { | ||||
return (&pv_invl_gen[pa_index(VM_PAGE_TO_PHYS(m)) % NPV_LIST_LOCKS]); | return (&pv_invl_gen[pa_index(VM_PAGE_TO_PHYS(m)) % NPV_LIST_LOCKS]); | ||||
} | } | ||||
static void | |||||
pmap_delayed_invl_callout_func(void *arg __unused) | |||||
{ | |||||
if (atomic_load_int(&pmap_invl_waiters) == 0) | |||||
return; | |||||
pmap_delayed_invl_finish_unblock(0); | |||||
} | |||||
static void | |||||
pmap_delayed_invl_callout_init(void *arg __unused) | |||||
{ | |||||
if (pmap_di_locked()) | |||||
return; | |||||
callout_init(&pmap_invl_callout, 1); | |||||
pmap_invl_callout_inited = true; | |||||
} | |||||
SYSINIT(pmap_di_callout, SI_SUB_CPU + 1, SI_ORDER_ANY, | |||||
pmap_delayed_invl_callout_init, NULL); | |||||
/* | /* | ||||
* Ensure that all currently executing DI blocks, that need to flush | * Ensure that all currently executing DI blocks, that need to flush | ||||
* TLB for the given page m, actually flushed the TLB at the time the | * TLB for the given page m, actually flushed the TLB at the time the | ||||
* function returned. If the page m has an empty PV list and we call | * function returned. If the page m has an empty PV list and we call | ||||
* pmap_delayed_invl_wait(), upon its return we know that no CPU has a | * pmap_delayed_invl_wait(), upon its return we know that no CPU has a | ||||
* valid mapping for the page m in either its page table or TLB. | * valid mapping for the page m in either its page table or TLB. | ||||
* | * | ||||
* This function works by blocking until the global DI generation | * This function works by blocking until the global DI generation | ||||
* number catches up with the generation number associated with the | * number catches up with the generation number associated with the | ||||
* given page m and its PV list. Since this function's callers | * given page m and its PV list. Since this function's callers | ||||
* typically own an object lock and sometimes own a page lock, it | * typically own an object lock and sometimes own a page lock, it | ||||
* cannot sleep. Instead, it blocks on a turnstile to relinquish the | * cannot sleep. Instead, it blocks on a turnstile to relinquish the | ||||
* processor. | * processor. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_delayed_invl_wait_l(vm_page_t m) | pmap_delayed_invl_wait_l(vm_page_t m) | ||||
{ | { | ||||
struct turnstile *ts; | |||||
u_long *m_gen; | u_long *m_gen; | ||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
bool accounted = false; | bool accounted = false; | ||||
#endif | #endif | ||||
m_gen = pmap_delayed_invl_genp(m); | m_gen = pmap_delayed_invl_genp(m); | ||||
while (*m_gen > pmap_invl_gen) { | while (*m_gen > pmap_invl_gen) { | ||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
if (!accounted) { | if (!accounted) { | ||||
atomic_add_long(&invl_wait, 1); | atomic_add_long(&invl_wait, 1); | ||||
accounted = true; | accounted = true; | ||||
} | } | ||||
#endif | #endif | ||||
ts = turnstile_trywait(&invl_gen_ts); | pmap_delayed_invl_wait_block(m_gen, &pmap_invl_gen); | ||||
if (*m_gen > pmap_invl_gen) | |||||
turnstile_wait(ts, NULL, TS_SHARED_QUEUE); | |||||
else | |||||
turnstile_cancel(ts); | |||||
} | } | ||||
} | } | ||||
static void | static void | ||||
pmap_delayed_invl_wait_u(vm_page_t m) | pmap_delayed_invl_wait_u(vm_page_t m) | ||||
{ | { | ||||
u_long *m_gen; | u_long *m_gen; | ||||
#ifdef PV_STATS | struct lock_delay_arg lda; | ||||
bool accounted = false; | bool fast; | ||||
#endif | |||||
fast = true; | |||||
m_gen = pmap_delayed_invl_genp(m); | m_gen = pmap_delayed_invl_genp(m); | ||||
lock_delay_arg_init(&lda, &di_delay); | |||||
while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) { | while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) { | ||||
#ifdef PV_STATS | if (fast || !pmap_invl_callout_inited) { | ||||
if (!accounted) { | PV_STAT(atomic_add_long(&invl_wait, 1)); | ||||
atomic_add_long(&invl_wait, 1); | lock_delay(&lda); | ||||
accounted = true; | fast = false; | ||||
} else { | |||||
atomic_add_int(&pmap_invl_waiters, 1); | |||||
if (*m_gen > | |||||
alc: should be "page's" | |||||
atomic_load_long(&pmap_invl_gen_head.gen)) { | |||||
markjUnsubmitted Not Done Inline ActionsWhy do you recheck the loop condition here? markj: Why do you recheck the loop condition here? | |||||
kibAuthorUnsubmitted Done Inline ActionsI do it to claim that the only race is due to the lost wakeup. We have in the thread finishing DI: store generation (atomic, works as seq consistent fence) load pmap_invl_waiters Without re-check, the order in the waiting thread is load generation increment pmap_invl_waiters (seq cst) which is reverse order to what is needed to make the increment fence to work against the store generation fence. With the re-check, the order becomes increment pmap_invl_waiters (seq cst) load generation You might argue that the callout would close this race anyway, but callout doing the real work means that the thread lost up to 1 tick of the wall time sitting blocked instead of making progress. So leaving callout use only to wakeup race proper is useful. kib: I do it to claim that the only race is due to the lost wakeup. We have in the thread… | |||||
markjUnsubmitted Not Done Inline ActionsThanks. I think this should be explained in a comment. markj: Thanks. I think this should be explained in a comment. | |||||
Done Inline ActionsI would say, "... below the current thread's number. Prepare to block so that we do not waste CPU cycles or worse, suffer livelock." alc: I would say, "... below the current thread's number. Prepare to block so that we do not waste… | |||||
callout_reset(&pmap_invl_callout, 1, | |||||
pmap_delayed_invl_callout_func, NULL); | |||||
PV_STAT(atomic_add_long(&invl_wait_slow, 1)); | |||||
pmap_delayed_invl_wait_block(m_gen, | |||||
Done Inline Actions"... impossible to block without ..." alc: "... impossible to block without ..." | |||||
&pmap_invl_gen_head.gen); | |||||
} | } | ||||
#endif | atomic_add_int(&pmap_invl_waiters, -1); | ||||
Done Inline Actions"... incrementing ..." alc: "... incrementing ..." | |||||
kern_yield(PRI_USER); | } | ||||
Not Done Inline ActionsVerb tense: "... which will unblock us if we lose ..." alc: Verb tense: "... which will unblock us if we lose ..." | |||||
} | } | ||||
} | } | ||||
DEFINE_IFUNC(, void, pmap_thread_init_invl_gen, (struct thread *)) | DEFINE_IFUNC(, void, pmap_thread_init_invl_gen, (struct thread *)) | ||||
{ | { | ||||
Done Inline Actions"... thread's invalidation ..." (Note the spelling correction to "invlidation".) alc: "... thread's invalidation ..." (Note the spelling correction to "invlidation".) | |||||
Not Done Inline Actions"Re-check the current ..." alc: "Re-check the current ..." | |||||
return (pmap_di_locked() ? pmap_thread_init_invl_gen_l : | return (pmap_di_locked() ? pmap_thread_init_invl_gen_l : | ||||
pmap_thread_init_invl_gen_u); | pmap_thread_init_invl_gen_u); | ||||
} | } | ||||
DEFINE_IFUNC(static, void, pmap_delayed_invl_start, (void)) | DEFINE_IFUNC(static, void, pmap_delayed_invl_start, (void)) | ||||
{ | { | ||||
return (pmap_di_locked() ? pmap_delayed_invl_start_l : | return (pmap_di_locked() ? pmap_delayed_invl_start_l : | ||||
▲ Show 20 Lines • Show All 8,911 Lines • Show Last 20 Lines |
should be "page's"