Index: sys/amd64/amd64/copyout.c =================================================================== --- sys/amd64/amd64/copyout.c +++ sys/amd64/amd64/copyout.c @@ -159,20 +159,41 @@ copyinstr_smap : copyinstr_nosmap); } -int copyin_nosmap(const void *udaddr, void *kaddr, size_t len); -int copyin_smap(const void *udaddr, void *kaddr, size_t len); +int copyin_nosmap_std(const void *udaddr, void *kaddr, size_t len); +int copyin_smap_std(const void *udaddr, void *kaddr, size_t len); +int copyin_nosmap_erms(const void *udaddr, void *kaddr, size_t len); +int copyin_smap_erms(const void *udaddr, void *kaddr, size_t len); DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t), static) { - return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? - copyin_smap : copyin_nosmap); + switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP|CPUID_STDEXT_ERMS)) { + case CPUID_STDEXT_SMAP: + return (copyin_smap_std); + case CPUID_STDEXT_ERMS: + return (copyin_nosmap_erms); + case CPUID_STDEXT_SMAP|CPUID_STDEXT_ERMS: + return (copyin_smap_erms); + default: + return (copyin_nosmap_std); + + } } -int copyout_nosmap(const void *kaddr, void *udaddr, size_t len); -int copyout_smap(const void *kaddr, void *udaddr, size_t len); +int copyout_nosmap_std(const void *kaddr, void *udaddr, size_t len); +int copyout_smap_std(const void *kaddr, void *udaddr, size_t len); +int copyout_nosmap_erms(const void *kaddr, void *udaddr, size_t len); +int copyout_smap_erms(const void *kaddr, void *udaddr, size_t len); DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t), static) { - return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? - copyout_smap : copyout_nosmap); + switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP|CPUID_STDEXT_ERMS)) { + case CPUID_STDEXT_SMAP: + return (copyout_smap_std); + case CPUID_STDEXT_ERMS: + return (copyout_nosmap_erms); + case CPUID_STDEXT_SMAP|CPUID_STDEXT_ERMS: + return (copyout_smap_erms); + default: + return (copyout_nosmap_std); + } } Index: sys/amd64/amd64/support.S =================================================================== --- sys/amd64/amd64/support.S +++ sys/amd64/amd64/support.S @@ -279,109 +279,88 @@ * These routines set curpcb->pcb_onfault for the time they execute. When a * protection violation occurs inside the functions, the trap handler * returns to *curpcb->pcb_onfault instead of the function. - */ - -/* - * copyout(from_kernel, to_user, len) - * %rdi, %rsi, %rdx - */ -ENTRY(copyout_nosmap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyout_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyout - - /* - * Check explicitly for non-user addresses. This check is essential - * because it prevents usermode from writing into the kernel. We do - * not verify anywhere else that the user did not specify a rogue - * address. - */ - /* - * First, prevent address wrapping. - */ - movq %rsi,%rax - addq %rdx,%rax - jc copyout_fault -/* - * XXX STOP USING VM_MAXUSER_ADDRESS. - * It is an end address, not a max, so every time it is used correctly it - * looks like there is an off by one error, and of course it caused an off - * by one error in several places. - */ - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyout_fault - - xchgq %rdi,%rsi - /* bcopy(%rsi, %rdi, %rdx) */ - movq %rdx,%rcx - - shrq $3,%rcx - rep - movsq - movb %dl,%cl - andb $7,%cl - je done_copyout - rep - movsb - - jmp done_copyout -END(copyout_nosmap) - -ENTRY(copyout_smap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - /* Trap entry clears PSL.AC */ - movq $copyout_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyout - - /* - * Check explicitly for non-user addresses. If 486 write protection - * is being used, this check is essential because we are in kernel - * mode so the h/w does not provide any protection against writing - * kernel addresses. - */ - - /* - * First, prevent address wrapping. - */ - movq %rsi,%rax - addq %rdx,%rax - jc copyout_fault -/* + * + * We check explicitly for non-user addresses. This check is essential + * because it prevents usermode from writing into the kernel. We do + * not verify anywhere else that the user did not specify a rogue + * address. + * * XXX STOP USING VM_MAXUSER_ADDRESS. * It is an end address, not a max, so every time it is used correctly it * looks like there is an off by one error, and of course it caused an off * by one error in several places. */ - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyout_fault - xchgq %rdi,%rsi - /* bcopy(%rsi, %rdi, %rdx) */ - movq %rdx,%rcx - - shrq $3,%rcx - stac - rep - movsq - movb %dl,%cl - andb $7,%cl - je 1f - rep - movsb -1: clac - -done_copyout: - xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) - POP_FRAME_POINTER - ret +#define SMAP_DISABLE stac +#define SMAP_ENABLE clac + +#define COPYOUT_PROLOGUE \ + PUSH_FRAME_POINTER; \ + movq PCPU(CURPCB),%rax; \ + movq $copyout_fault,PCB_ONFAULT(%rax); \ + testq %rdx,%rdx; \ + jz 1f; \ + movq %rsi,%rax; \ + addq %rdx,%rax; \ + jc copyout_fault; \ + movq %rsi,%rax; \ + addq %rdx,%rax; \ + jc copyout_fault; \ + movq $VM_MAXUSER_ADDRESS,%rcx; \ + cmpq %rcx,%rax; \ + ja copyout_fault; \ + xchgq %rdi,%rsi; \ + movq %rdx,%rcx; + +#define COPYOUT_COPY_STD \ + shrq $3,%rcx; \ + rep \ + movsq; \ + movb %dl,%cl; \ + andb $7,%cl; \ + je 1f; \ + rep \ + movsb; + +#define COPYOUT_COPY_ERMS \ + rep \ + movsb; + +#define COPYOUT_EPILOGUE \ +1: \ + xorl %eax,%eax; \ + movq PCPU(CURPCB),%rdx; \ + movq %rax,PCB_ONFAULT(%rdx); \ + POP_FRAME_POINTER; \ + ret; + +ENTRY(copyout_nosmap_std) + COPYOUT_PROLOGUE + COPYOUT_COPY_STD + COPYOUT_EPILOGUE +END(copyout_nosmap_std) + +ENTRY(copyout_smap_std) + COPYOUT_PROLOGUE + SMAP_DISABLE + COPYOUT_COPY_STD + SMAP_ENABLE + COPYOUT_EPILOGUE +END(copyout_smap_std) + +ENTRY(copyout_nosmap_erms) + COPYOUT_PROLOGUE + COPYOUT_COPY_ERMS + COPYOUT_EPILOGUE +END(copyout_nosmap_erms) + +ENTRY(copyout_smap_erms) + COPYOUT_PROLOGUE + SMAP_DISABLE + COPYOUT_COPY_ERMS + SMAP_ENABLE + COPYOUT_EPILOGUE +END(copyout_smap_erms) ALIGN_TEXT copyout_fault: @@ -390,82 +369,72 @@ movq $EFAULT,%rax POP_FRAME_POINTER ret -END(copyout_smap) - -/* - * copyin(from_user, to_kernel, len) - * %rdi, %rsi, %rdx - */ -ENTRY(copyin_nosmap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyin_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyin - - /* - * make sure address is valid - */ - movq %rdi,%rax - addq %rdx,%rax - jc copyin_fault - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyin_fault - xchgq %rdi,%rsi - movq %rdx,%rcx - movb %cl,%al - shrq $3,%rcx /* copy longword-wise */ - rep - movsq - movb %al,%cl - andb $7,%cl /* copy remaining bytes */ - je done_copyin - rep - movsb - - jmp done_copyin -END(copyin_nosmap) - -ENTRY(copyin_smap) - PUSH_FRAME_POINTER - movq PCPU(CURPCB),%rax - movq $copyin_fault,PCB_ONFAULT(%rax) - testq %rdx,%rdx /* anything to do? */ - jz done_copyin - - /* - * make sure address is valid - */ - movq %rdi,%rax - addq %rdx,%rax - jc copyin_fault - movq $VM_MAXUSER_ADDRESS,%rcx - cmpq %rcx,%rax - ja copyin_fault - - xchgq %rdi,%rsi - movq %rdx,%rcx - movb %cl,%al - shrq $3,%rcx /* copy longword-wise */ - stac - rep - movsq - movb %al,%cl - andb $7,%cl /* copy remaining bytes */ - je 1f - rep - movsb -1: clac - -done_copyin: - xorl %eax,%eax - movq PCPU(CURPCB),%rdx - movq %rax,PCB_ONFAULT(%rdx) - POP_FRAME_POINTER - ret -END(copyin_smap) +#define COPYIN_PROLOGUE \ + PUSH_FRAME_POINTER; \ + movq PCPU(CURPCB),%rax; \ + movq $copyin_fault,PCB_ONFAULT(%rax); \ + testq %rdx,%rdx; \ + jz 1f; \ + movq %rdi,%rax; \ + addq %rdx,%rax; \ + jc copyin_fault; \ + movq $VM_MAXUSER_ADDRESS,%rcx; \ + cmpq %rcx,%rax; \ + ja copyin_fault; \ + xchgq %rdi,%rsi; \ + movq %rdx,%rcx; \ + movb %cl,%al; + +#define COPYIN_COPY_STD \ + shrq $3,%rcx; \ + rep \ + movsq; \ + movb %al,%cl; \ + andb $7,%cl; \ + je 1f; \ + rep \ + movsb; + +#define COPYIN_COPY_ERMS \ + rep \ + movsb; + +#define COPYIN_EPILOGUE \ +1: \ + xorl %eax,%eax; \ + movq PCPU(CURPCB),%rdx; \ + movq %rax,PCB_ONFAULT(%rdx); \ + POP_FRAME_POINTER; \ + ret; + +ENTRY(copyin_nosmap_std) + COPYIN_PROLOGUE + COPYIN_COPY_STD + COPYIN_EPILOGUE +END(copyin_nosmap_std) + +ENTRY(copyin_smap_std) + COPYIN_PROLOGUE + SMAP_DISABLE + COPYIN_COPY_STD + SMAP_ENABLE + COPYIN_EPILOGUE +END(copyin_smap_std) + +ENTRY(copyin_nosmap_erms) + COPYIN_PROLOGUE + COPYIN_COPY_ERMS + COPYIN_EPILOGUE +END(copyin_nosmap_erms) + +ENTRY(copyin_smap_erms) + COPYIN_PROLOGUE + SMAP_DISABLE + COPYIN_COPY_ERMS + SMAP_ENABLE + COPYIN_EPILOGUE +END(copyin_smap_erms) ALIGN_TEXT copyin_fault: