Changeset View
Changeset View
Standalone View
Standalone View
head/sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
| Show First 20 Lines • Show All 506 Lines • ▼ Show 20 Lines | sysctl_pmap_di_locked(SYSCTL_HANDLER_ARGS) | ||||
| return (sysctl_handle_int(oidp, &locked, 0, req)); | return (sysctl_handle_int(oidp, &locked, 0, req)); | ||||
| } | } | ||||
| SYSCTL_PROC(_vm_pmap, OID_AUTO, di_locked, CTLTYPE_INT | CTLFLAG_RDTUN | | SYSCTL_PROC(_vm_pmap, OID_AUTO, di_locked, CTLTYPE_INT | CTLFLAG_RDTUN | | ||||
| CTLFLAG_MPSAFE, 0, 0, sysctl_pmap_di_locked, "", | CTLFLAG_MPSAFE, 0, 0, sysctl_pmap_di_locked, "", | ||||
| "Locked delayed invalidation"); | "Locked delayed invalidation"); | ||||
| static bool pmap_not_in_di_l(void); | static bool pmap_not_in_di_l(void); | ||||
| static bool pmap_not_in_di_u(void); | static bool pmap_not_in_di_u(void); | ||||
| DEFINE_IFUNC(, bool, pmap_not_in_di, (void), static) | DEFINE_IFUNC(, bool, pmap_not_in_di, (void)) | ||||
| { | { | ||||
| return (pmap_di_locked() ? pmap_not_in_di_l : pmap_not_in_di_u); | return (pmap_di_locked() ? pmap_not_in_di_l : pmap_not_in_di_u); | ||||
| } | } | ||||
| static bool | static bool | ||||
| pmap_not_in_di_l(void) | pmap_not_in_di_l(void) | ||||
| { | { | ||||
| ▲ Show 20 Lines • Show All 424 Lines • ▼ Show 20 Lines | if (!accounted) { | ||||
| atomic_add_long(&invl_wait, 1); | atomic_add_long(&invl_wait, 1); | ||||
| accounted = true; | accounted = true; | ||||
| } | } | ||||
| #endif | #endif | ||||
| kern_yield(PRI_USER); | kern_yield(PRI_USER); | ||||
| } | } | ||||
| } | } | ||||
| DEFINE_IFUNC(, void, pmap_thread_init_invl_gen, (struct thread *), static) | DEFINE_IFUNC(, void, pmap_thread_init_invl_gen, (struct thread *)) | ||||
| { | { | ||||
| return (pmap_di_locked() ? pmap_thread_init_invl_gen_l : | return (pmap_di_locked() ? pmap_thread_init_invl_gen_l : | ||||
| pmap_thread_init_invl_gen_u); | pmap_thread_init_invl_gen_u); | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_delayed_invl_start, (void), static) | DEFINE_IFUNC(static, void, pmap_delayed_invl_start, (void)) | ||||
| { | { | ||||
| return (pmap_di_locked() ? pmap_delayed_invl_start_l : | return (pmap_di_locked() ? pmap_delayed_invl_start_l : | ||||
| pmap_delayed_invl_start_u); | pmap_delayed_invl_start_u); | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_delayed_invl_finish, (void), static) | DEFINE_IFUNC(static, void, pmap_delayed_invl_finish, (void)) | ||||
| { | { | ||||
| return (pmap_di_locked() ? pmap_delayed_invl_finish_l : | return (pmap_di_locked() ? pmap_delayed_invl_finish_l : | ||||
| pmap_delayed_invl_finish_u); | pmap_delayed_invl_finish_u); | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_delayed_invl_wait, (vm_page_t), static) | DEFINE_IFUNC(static, void, pmap_delayed_invl_wait, (vm_page_t)) | ||||
| { | { | ||||
| return (pmap_di_locked() ? pmap_delayed_invl_wait_l : | return (pmap_di_locked() ? pmap_delayed_invl_wait_l : | ||||
| pmap_delayed_invl_wait_u); | pmap_delayed_invl_wait_u); | ||||
| } | } | ||||
| /* | /* | ||||
| * Mark the page m's PV list as participating in the current thread's | * Mark the page m's PV list as participating in the current thread's | ||||
| ▲ Show 20 Lines • Show All 1,206 Lines • ▼ Show 20 Lines | pmap_invalidate_page_pcid_noinvpcid(pmap_t pmap, vm_offset_t va) | ||||
| pmap_invalidate_page_pcid(pmap, va, false); | pmap_invalidate_page_pcid(pmap, va, false); | ||||
| } | } | ||||
| static void | static void | ||||
| pmap_invalidate_page_nopcid(pmap_t pmap, vm_offset_t va) | pmap_invalidate_page_nopcid(pmap_t pmap, vm_offset_t va) | ||||
| { | { | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_invalidate_page_mode, (pmap_t, vm_offset_t), | DEFINE_IFUNC(static, void, pmap_invalidate_page_mode, (pmap_t, vm_offset_t)) | ||||
| static) | |||||
| { | { | ||||
| if (pmap_pcid_enabled) | if (pmap_pcid_enabled) | ||||
| return (invpcid_works ? pmap_invalidate_page_pcid_invpcid : | return (invpcid_works ? pmap_invalidate_page_pcid_invpcid : | ||||
| pmap_invalidate_page_pcid_noinvpcid); | pmap_invalidate_page_pcid_noinvpcid); | ||||
| return (pmap_invalidate_page_nopcid); | return (pmap_invalidate_page_nopcid); | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | |||||
| } | } | ||||
| static void | static void | ||||
| pmap_invalidate_range_nopcid(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_invalidate_range_nopcid(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
| { | { | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_invalidate_range_mode, (pmap_t, vm_offset_t, | DEFINE_IFUNC(static, void, pmap_invalidate_range_mode, (pmap_t, vm_offset_t, | ||||
| vm_offset_t), static) | vm_offset_t)) | ||||
| { | { | ||||
| if (pmap_pcid_enabled) | if (pmap_pcid_enabled) | ||||
| return (invpcid_works ? pmap_invalidate_range_pcid_invpcid : | return (invpcid_works ? pmap_invalidate_range_pcid_invpcid : | ||||
| pmap_invalidate_range_pcid_noinvpcid); | pmap_invalidate_range_pcid_noinvpcid); | ||||
| return (pmap_invalidate_range_nopcid); | return (pmap_invalidate_range_nopcid); | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | |||||
| { | { | ||||
| if (pmap == kernel_pmap) | if (pmap == kernel_pmap) | ||||
| invltlb_glob(); | invltlb_glob(); | ||||
| else if (pmap == PCPU_GET(curpmap)) | else if (pmap == PCPU_GET(curpmap)) | ||||
| invltlb(); | invltlb(); | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_invalidate_all_mode, (pmap_t), static) | DEFINE_IFUNC(static, void, pmap_invalidate_all_mode, (pmap_t)) | ||||
| { | { | ||||
| if (pmap_pcid_enabled) | if (pmap_pcid_enabled) | ||||
| return (invpcid_works ? pmap_invalidate_all_pcid_invpcid : | return (invpcid_works ? pmap_invalidate_all_pcid_invpcid : | ||||
| pmap_invalidate_all_pcid_noinvpcid); | pmap_invalidate_all_pcid_noinvpcid); | ||||
| return (pmap_invalidate_all_nopcid); | return (pmap_invalidate_all_nopcid); | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 262 Lines • ▼ Show 20 Lines | pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde) | ||||
| */ | */ | ||||
| if ((pde & PG_PROMOTED) != 0) | if ((pde & PG_PROMOTED) != 0) | ||||
| pmap_invalidate_range(pmap, va, va + NBPDR - 1); | pmap_invalidate_range(pmap, va, va + NBPDR - 1); | ||||
| else | else | ||||
| pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
| } | } | ||||
| DEFINE_IFUNC(, void, pmap_invalidate_cache_range, | DEFINE_IFUNC(, void, pmap_invalidate_cache_range, | ||||
| (vm_offset_t sva, vm_offset_t eva), static) | (vm_offset_t sva, vm_offset_t eva)) | ||||
| { | { | ||||
| if ((cpu_feature & CPUID_SS) != 0) | if ((cpu_feature & CPUID_SS) != 0) | ||||
| return (pmap_invalidate_cache_range_selfsnoop); | return (pmap_invalidate_cache_range_selfsnoop); | ||||
| if ((cpu_feature & CPUID_CLFSH) != 0) | if ((cpu_feature & CPUID_CLFSH) != 0) | ||||
| return (pmap_force_invalidate_cache_range); | return (pmap_force_invalidate_cache_range); | ||||
| return (pmap_invalidate_cache_range_all); | return (pmap_invalidate_cache_range_all); | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 5,576 Lines • ▼ Show 20 Lines | pmap_activate_sw_nopcid_pti(struct thread *td, pmap_t pmap, | ||||
| pmap_activate_sw_nopcid_nopti(td, pmap, cpuid); | pmap_activate_sw_nopcid_nopti(td, pmap, cpuid); | ||||
| PCPU_SET(kcr3, pmap->pm_cr3); | PCPU_SET(kcr3, pmap->pm_cr3); | ||||
| PCPU_SET(ucr3, pmap->pm_ucr3); | PCPU_SET(ucr3, pmap->pm_ucr3); | ||||
| pmap_activate_sw_pti_post(td, pmap); | pmap_activate_sw_pti_post(td, pmap); | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_activate_sw_mode, (struct thread *, pmap_t, | DEFINE_IFUNC(static, void, pmap_activate_sw_mode, (struct thread *, pmap_t, | ||||
| u_int), static) | u_int)) | ||||
| { | { | ||||
| if (pmap_pcid_enabled && pti && invpcid_works) | if (pmap_pcid_enabled && pti && invpcid_works) | ||||
| return (pmap_activate_sw_pcid_invpcid_pti); | return (pmap_activate_sw_pcid_invpcid_pti); | ||||
| else if (pmap_pcid_enabled && pti && !invpcid_works) | else if (pmap_pcid_enabled && pti && !invpcid_works) | ||||
| return (pmap_activate_sw_pcid_noinvpcid_pti); | return (pmap_activate_sw_pcid_noinvpcid_pti); | ||||
| else if (pmap_pcid_enabled && !pti && invpcid_works) | else if (pmap_pcid_enabled && !pti && invpcid_works) | ||||
| return (pmap_activate_sw_pcid_nopti); | return (pmap_activate_sw_pcid_nopti); | ||||
| ▲ Show 20 Lines • Show All 680 Lines • ▼ Show 20 Lines | pmap_large_map_wb_fence_sfence(void) | ||||
| sfence(); | sfence(); | ||||
| } | } | ||||
| static void | static void | ||||
| pmap_large_map_wb_fence_nop(void) | pmap_large_map_wb_fence_nop(void) | ||||
| { | { | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_large_map_wb_fence, (void), static) | DEFINE_IFUNC(static, void, pmap_large_map_wb_fence, (void)) | ||||
| { | { | ||||
| if (cpu_vendor_id != CPU_VENDOR_INTEL) | if (cpu_vendor_id != CPU_VENDOR_INTEL) | ||||
| return (pmap_large_map_wb_fence_mfence); | return (pmap_large_map_wb_fence_mfence); | ||||
| else if ((cpu_stdext_feature & (CPUID_STDEXT_CLWB | | else if ((cpu_stdext_feature & (CPUID_STDEXT_CLWB | | ||||
| CPUID_STDEXT_CLFLUSHOPT)) == 0) | CPUID_STDEXT_CLFLUSHOPT)) == 0) | ||||
| return (pmap_large_map_wb_fence_sfence); | return (pmap_large_map_wb_fence_sfence); | ||||
| else | else | ||||
| Show All 28 Lines | for (; len > 0; len -= cpu_clflush_line_size, | ||||
| clflush(va); | clflush(va); | ||||
| } | } | ||||
| static void | static void | ||||
| pmap_large_map_flush_range_nop(vm_offset_t sva __unused, vm_size_t len __unused) | pmap_large_map_flush_range_nop(vm_offset_t sva __unused, vm_size_t len __unused) | ||||
| { | { | ||||
| } | } | ||||
| DEFINE_IFUNC(static, void, pmap_large_map_flush_range, (vm_offset_t, vm_size_t), | DEFINE_IFUNC(static, void, pmap_large_map_flush_range, (vm_offset_t, vm_size_t)) | ||||
| static) | |||||
| { | { | ||||
| if ((cpu_stdext_feature & CPUID_STDEXT_CLWB) != 0) | if ((cpu_stdext_feature & CPUID_STDEXT_CLWB) != 0) | ||||
| return (pmap_large_map_flush_range_clwb); | return (pmap_large_map_flush_range_clwb); | ||||
| else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) | else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) | ||||
| return (pmap_large_map_flush_range_clflushopt); | return (pmap_large_map_flush_range_clflushopt); | ||||
| else if ((cpu_feature & CPUID_CLFSH) != 0) | else if ((cpu_feature & CPUID_CLFSH) != 0) | ||||
| return (pmap_large_map_flush_range_clflush); | return (pmap_large_map_flush_range_clflush); | ||||
| ▲ Show 20 Lines • Show All 728 Lines • Show Last 20 Lines | |||||