Changeset View
Changeset View
Standalone View
Standalone View
sys/i386/i386/copyout.c
Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/vm_extern.h> | #include <vm/vm_extern.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <vm/vm_map.h> | #include <vm/vm_map.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#if defined(PAE) || defined(PAE_TABLES) | |||||
#define KCR3 ((u_int)IdlePDPT) | |||||
#else | |||||
#define KCR3 ((u_int)IdlePTD) | |||||
#endif | |||||
int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int); | int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int); | ||||
static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int); | static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int); | ||||
int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int); | int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int); | ||||
static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int); | static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int); | ||||
int fubyte_fast(volatile const void *base, u_int kcr3); | int fubyte_fast(volatile const void *base, u_int kcr3); | ||||
static int (*fubyte_fast_tramp)(volatile const void *, u_int); | static int (*fubyte_fast_tramp)(volatile const void *, u_int); | ||||
int fuword16_fast(volatile const void *base, u_int kcr3); | int fuword16_fast(volatile const void *base, u_int kcr3); | ||||
static int (*fuword16_fast_tramp)(volatile const void *, u_int); | static int (*fuword16_fast_tramp)(volatile const void *, u_int); | ||||
Show All 34 Lines | |||||
} | } | ||||
int | int | ||||
cp_slow0(vm_offset_t uva, size_t len, bool write, | cp_slow0(vm_offset_t uva, size_t len, bool write, | ||||
void (*f)(vm_offset_t, void *), void *arg) | void (*f)(vm_offset_t, void *), void *arg) | ||||
{ | { | ||||
struct pcpu *pc; | struct pcpu *pc; | ||||
vm_page_t m[2]; | vm_page_t m[2]; | ||||
pt_entry_t *pte; | |||||
vm_offset_t kaddr; | vm_offset_t kaddr; | ||||
int error, i, plen; | int error, i, plen; | ||||
bool sleepable; | bool sleepable; | ||||
plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE); | plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE); | ||||
MPASS(plen <= nitems(m)); | MPASS(plen <= nitems(m)); | ||||
error = 0; | error = 0; | ||||
i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len, | i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len, | ||||
(write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT, | (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT, | ||||
m, nitems(m)); | m, nitems(m)); | ||||
if (i != plen) | if (i != plen) | ||||
return (EFAULT); | return (EFAULT); | ||||
sched_pin(); | sched_pin(); | ||||
pc = get_pcpu(); | pc = get_pcpu(); | ||||
if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 || | if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 || | ||||
(curthread->td_pflags & TDP_NOFAULTING) != 0) { | (curthread->td_pflags & TDP_NOFAULTING) != 0) { | ||||
sleepable = false; | sleepable = false; | ||||
mtx_lock(&pc->pc_copyout_mlock); | mtx_lock(&pc->pc_copyout_mlock); | ||||
kaddr = pc->pc_copyout_maddr; | kaddr = pc->pc_copyout_maddr; | ||||
} else { | } else { | ||||
sleepable = true; | sleepable = true; | ||||
sx_xlock(&pc->pc_copyout_slock); | sx_xlock(&pc->pc_copyout_slock); | ||||
kaddr = pc->pc_copyout_saddr; | kaddr = pc->pc_copyout_saddr; | ||||
} | } | ||||
for (i = 0, pte = vtopte(kaddr); i < plen; i++, pte++) { | pmap_cp_slow0_map(kaddr, plen, m); | ||||
*pte = PG_V | PG_RW | PG_A | PG_M | VM_PAGE_TO_PHYS(m[i]) | | |||||
pmap_cache_bits(kernel_pmap, pmap_page_get_memattr(m[i]), | |||||
FALSE); | |||||
invlpg(kaddr + ptoa(i)); | |||||
} | |||||
kaddr += uva - trunc_page(uva); | kaddr += uva - trunc_page(uva); | ||||
f(kaddr, arg); | f(kaddr, arg); | ||||
sched_unpin(); | sched_unpin(); | ||||
if (sleepable) | if (sleepable) | ||||
sx_xunlock(&pc->pc_copyout_slock); | sx_xunlock(&pc->pc_copyout_slock); | ||||
else | else | ||||
mtx_unlock(&pc->pc_copyout_mlock); | mtx_unlock(&pc->pc_copyout_mlock); | ||||
vm_page_unhold_pages(m, plen); | vm_page_unhold_pages(m, plen); | ||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | copyin(const void *udaddr, void *kaddr, size_t len) | ||||
struct copyin_arg0 ca; | struct copyin_arg0 ca; | ||||
vm_offset_t uc; | vm_offset_t uc; | ||||
size_t plen; | size_t plen; | ||||
if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || | if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || | ||||
(uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) | (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && | if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && | ||||
copyin_fast_tramp(udaddr, kaddr, len, KCR3) == 0)) | copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0)) | ||||
return (0); | return (0); | ||||
for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; | for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; | ||||
plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { | plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { | ||||
ca.len = round_page(uc) - uc; | ca.len = round_page(uc) - uc; | ||||
if (ca.len == 0) | if (ca.len == 0) | ||||
ca.len = PAGE_SIZE; | ca.len = PAGE_SIZE; | ||||
if (plen + ca.len > len) | if (plen + ca.len > len) | ||||
ca.len = len - plen; | ca.len = len - plen; | ||||
Show All 18 Lines | copyout(const void *kaddr, void *udaddr, size_t len) | ||||
struct copyin_arg0 ca; | struct copyin_arg0 ca; | ||||
vm_offset_t uc; | vm_offset_t uc; | ||||
size_t plen; | size_t plen; | ||||
if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || | if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || | ||||
(uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) | (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) | ||||
return (EFAULT); | return (EFAULT); | ||||
if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && | if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && | ||||
copyout_fast_tramp(kaddr, udaddr, len, KCR3) == 0)) | copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0)) | ||||
return (0); | return (0); | ||||
for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; | for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; | ||||
plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { | plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { | ||||
ca.len = round_page(uc) - uc; | ca.len = round_page(uc) - uc; | ||||
if (ca.len == 0) | if (ca.len == 0) | ||||
ca.len = PAGE_SIZE; | ca.len = PAGE_SIZE; | ||||
if (plen + ca.len > len) | if (plen + ca.len > len) | ||||
ca.len = len - plen; | ca.len = len - plen; | ||||
Show All 19 Lines | |||||
fubyte(volatile const void *base) | fubyte(volatile const void *base) | ||||
{ | { | ||||
int res; | int res; | ||||
if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout) { | if (fast_copyout) { | ||||
res = fubyte_fast_tramp(base, KCR3); | res = fubyte_fast_tramp(base, pmap_get_kcr3()); | ||||
if (res != -1) | if (res != -1) | ||||
return (res); | return (res); | ||||
} | } | ||||
if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0, | if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0, | ||||
&res) != 0) | &res) != 0) | ||||
return (-1); | return (-1); | ||||
return (res); | return (res); | ||||
} | } | ||||
Show All 9 Lines | |||||
fuword16(volatile const void *base) | fuword16(volatile const void *base) | ||||
{ | { | ||||
int res; | int res; | ||||
if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout) { | if (fast_copyout) { | ||||
res = fuword16_fast_tramp(base, KCR3); | res = fuword16_fast_tramp(base, pmap_get_kcr3()); | ||||
if (res != -1) | if (res != -1) | ||||
return (res); | return (res); | ||||
} | } | ||||
if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false, | if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false, | ||||
fuword16_slow0, &res) != 0) | fuword16_slow0, &res) != 0) | ||||
return (-1); | return (-1); | ||||
return (res); | return (res); | ||||
} | } | ||||
Show All 9 Lines | |||||
fueword(volatile const void *base, long *val) | fueword(volatile const void *base, long *val) | ||||
{ | { | ||||
uint32_t res; | uint32_t res; | ||||
if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout) { | if (fast_copyout) { | ||||
if (fueword_fast_tramp(base, val, KCR3) == 0) | if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0) | ||||
return (0); | return (0); | ||||
} | } | ||||
if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0, | if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0, | ||||
&res) != 0) | &res) != 0) | ||||
return (-1); | return (-1); | ||||
*val = res; | *val = res; | ||||
return (0); | return (0); | ||||
} | } | ||||
Show All 18 Lines | |||||
int | int | ||||
subyte(volatile void *base, int byte) | subyte(volatile void *base, int byte) | ||||
{ | { | ||||
if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout && subyte_fast_tramp(base, byte, KCR3) == 0) | if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0) | ||||
return (0); | return (0); | ||||
return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0, | return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0, | ||||
&byte) != 0 ? -1 : 0); | &byte) != 0 ? -1 : 0); | ||||
} | } | ||||
static void | static void | ||||
suword16_slow0(vm_offset_t kva, void *arg) | suword16_slow0(vm_offset_t kva, void *arg) | ||||
{ | { | ||||
*(int *)kva = *(uint16_t *)arg; | *(int *)kva = *(uint16_t *)arg; | ||||
} | } | ||||
int | int | ||||
suword16(volatile void *base, int word) | suword16(volatile void *base, int word) | ||||
{ | { | ||||
if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout && suword16_fast_tramp(base, word, KCR3) == 0) | if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3()) | ||||
== 0) | |||||
return (0); | return (0); | ||||
return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true, | return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true, | ||||
suword16_slow0, &word) != 0 ? -1 : 0); | suword16_slow0, &word) != 0 ? -1 : 0); | ||||
} | } | ||||
static void | static void | ||||
suword_slow0(vm_offset_t kva, void *arg) | suword_slow0(vm_offset_t kva, void *arg) | ||||
{ | { | ||||
*(int *)kva = *(uint32_t *)arg; | *(int *)kva = *(uint32_t *)arg; | ||||
} | } | ||||
int | int | ||||
suword(volatile void *base, long word) | suword(volatile void *base, long word) | ||||
{ | { | ||||
if ((uintptr_t)base + sizeof(word) < (uintptr_t)base || | if ((uintptr_t)base + sizeof(word) < (uintptr_t)base || | ||||
(uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS) | (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS) | ||||
return (-1); | return (-1); | ||||
if (fast_copyout && suword_fast_tramp(base, word, KCR3) == 0) | if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0) | ||||
return (0); | return (0); | ||||
return (cp_slow0((vm_offset_t)base, sizeof(long), true, | return (cp_slow0((vm_offset_t)base, sizeof(long), true, | ||||
suword_slow0, &word) != 0 ? -1 : 0); | suword_slow0, &word) != 0 ? -1 : 0); | ||||
} | } | ||||
int | int | ||||
suword32(volatile void *base, int32_t word) | suword32(volatile void *base, int32_t word) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 52 Lines • Show Last 20 Lines |