Index: head/sys/amd64/amd64/copyout.c =================================================================== --- head/sys/amd64/amd64/copyout.c +++ head/sys/amd64/amd64/copyout.c @@ -159,20 +159,41 @@ copyinstr_smap : copyinstr_nosmap); } -int copyin_nosmap(const void *udaddr, void *kaddr, size_t len); -int copyin_smap(const void *udaddr, void *kaddr, size_t len); +int copyin_nosmap_std(const void *udaddr, void *kaddr, size_t len); +int copyin_smap_std(const void *udaddr, void *kaddr, size_t len); +int copyin_nosmap_erms(const void *udaddr, void *kaddr, size_t len); +int copyin_smap_erms(const void *udaddr, void *kaddr, size_t len); DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t), static) { - return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? - copyin_smap : copyin_nosmap); + switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { + case CPUID_STDEXT_SMAP: + return (copyin_smap_std); + case CPUID_STDEXT_ERMS: + return (copyin_nosmap_erms); + case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: + return (copyin_smap_erms); + default: + return (copyin_nosmap_std); + + } } -int copyout_nosmap(const void *kaddr, void *udaddr, size_t len); -int copyout_smap(const void *kaddr, void *udaddr, size_t len); +int copyout_nosmap_std(const void *kaddr, void *udaddr, size_t len); +int copyout_smap_std(const void *kaddr, void *udaddr, size_t len); +int copyout_nosmap_erms(const void *kaddr, void *udaddr, size_t len); +int copyout_smap_erms(const void *kaddr, void *udaddr, size_t len); DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t), static) { - return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? - copyout_smap : copyout_nosmap); + switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { + case CPUID_STDEXT_SMAP: + return (copyout_smap_std); + case CPUID_STDEXT_ERMS: + return (copyout_nosmap_erms); + case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: + return (copyout_smap_erms); + default: + return (copyout_nosmap_std); + } } Index: head/sys/amd64/amd64/support.S =================================================================== --- head/sys/amd64/amd64/support.S +++ head/sys/amd64/amd64/support.S @@ -281,62 +281,30 @@ * returns to *curpcb->pcb_onfault instead of the function. */ +.macro SMAP_DISABLE smap +.if \smap + stac +.endif +.endmacro + + +.macro SMAP_ENABLE smap +.if \smap + clac +.endif +.endmacro + /* * copyout(from_kernel, to_user, len) * %rdi, %rsi, %rdx */ -ENTRY(copyout_nosmap) +.macro COPYOUT smap erms PUSH_FRAME_POINTER movq PCPU(CURPCB),%rax - movq $copyout_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyout - - /* - * Check explicitly for non-user addresses. This check is essential - * because it prevents usermode from writing into the kernel. We do - * not verify anywhere else that the user did not specify a rogue - * address. - */ - /* - * First, prevent address wrapping. - */ - movq %rsi,%rax - addq %rdx,%rax - jc copyout_fault -/* - * XXX STOP USING VM_MAXUSER_ADDRESS. - * It is an end address, not a max, so every time it is used correctly it - * looks like there is an off by one error, and of course it caused an off - * by one error in several places. - */ - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyout_fault - - xchgq %rdi,%rsi - /* bcopy(%rsi, %rdi, %rdx) */ - movq %rdx,%rcx - - shrq $3,%rcx - rep - movsq - movb %dl,%cl - andb $7,%cl - je done_copyout - rep - movsb - - jmp done_copyout -END(copyout_nosmap) - -ENTRY(copyout_smap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax /* Trap entry clears PSL.AC */ movq $copyout_fault,PCB_ONFAULT(%rax) testq %rdx,%rdx /* anything to do? */ - jz done_copyout + jz 2f /* * Check explicitly for non-user addresses. If 486 write protection @@ -365,24 +333,43 @@ /* bcopy(%rsi, %rdi, %rdx) */ movq %rdx,%rcx + SMAP_DISABLE \smap +.if \erms == 0 shrq $3,%rcx - stac rep movsq movb %dl,%cl andb $7,%cl je 1f +.endif rep movsb -1: clac - -done_copyout: +1: + SMAP_ENABLE \smap +2: xorl %eax,%eax movq PCPU(CURPCB),%rdx movq %rax,PCB_ONFAULT(%rdx) POP_FRAME_POINTER ret +.endmacro +ENTRY(copyout_nosmap_std) + COPYOUT smap=0 erms=0 +END(copyout_nosmap_std) + +ENTRY(copyout_smap_std) + COPYOUT smap=1 erms=0 +END(copyout_smap_std) + +ENTRY(copyout_nosmap_erms) + COPYOUT smap=0 erms=1 +END(copyout_nosmap_erms) + +ENTRY(copyout_smap_erms) + COPYOUT smap=1 erms=1 +END(copyout_smap_erms) + ALIGN_TEXT copyout_fault: movq PCPU(CURPCB),%rdx @@ -390,18 +377,17 @@ movq $EFAULT,%rax POP_FRAME_POINTER ret -END(copyout_smap) /* * copyin(from_user, to_kernel, len) * %rdi, %rsi, %rdx */ -ENTRY(copyin_nosmap) +.macro COPYIN smap erms PUSH_FRAME_POINTER movq PCPU(CURPCB),%rax movq $copyin_fault,PCB_ONFAULT(%rax) testq %rdx,%rdx /* anything to do? */ - jz done_copyin + jz 2f /* * make sure address is valid @@ -416,56 +402,44 @@ xchgq %rdi,%rsi movq %rdx,%rcx movb %cl,%al - shrq $3,%rcx /* copy longword-wise */ - rep - movsq - movb %al,%cl - andb $7,%cl /* copy remaining bytes */ - je done_copyin - rep - movsb - jmp done_copyin -END(copyin_nosmap) - -ENTRY(copyin_smap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyin_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyin - - /* - * make sure address is valid - */ - movq %rdi,%rax - addq %rdx,%rax - jc copyin_fault - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyin_fault - - xchgq %rdi,%rsi - movq %rdx,%rcx - movb %cl,%al + SMAP_DISABLE \smap +.if \erms == 0 shrq $3,%rcx /* copy longword-wise */ - stac rep movsq movb %al,%cl andb $7,%cl /* copy remaining bytes */ - je 1f + je 1 +.endif rep movsb -1: clac -done_copyin: +1: + SMAP_ENABLE \smap +2: xorl %eax,%eax movq PCPU(CURPCB),%rdx movq %rax,PCB_ONFAULT(%rdx) POP_FRAME_POINTER ret -END(copyin_smap) +.endmacro + +ENTRY(copyin_nosmap_std) + COPYIN smap=0 erms=0 +END(copyin_nosmap_std) + +ENTRY(copyin_smap_std) + COPYIN smap=1 erms=0 +END(copyin_smap_std) + +ENTRY(copyin_nosmap_erms) + COPYIN smap=0 erms=1 +END(copyin_nosmap_erms) + +ENTRY(copyin_smap_erms) + COPYIN smap=1 erms=1 +END(copyin_smap_erms) ALIGN_TEXT copyin_fault: