Differential D13838 Diff 37796 sys/amd64/amd64/support.S

Changeset View

Standalone View

sys/amd64/amd64/support.S

Show First 20 Lines • Show All 230 Lines • ▼ Show 20 Lines
* protection violation occurs inside the functions, the trap handler		* protection violation occurs inside the functions, the trap handler
* returns to *curpcb->pcb_onfault instead of the function.		* returns to *curpcb->pcb_onfault instead of the function.
*/		*/

/*		/*
* copyout(from_kernel, to_user, len)		* copyout(from_kernel, to_user, len)
* %rdi, %rsi, %rdx		* %rdi, %rsi, %rdx
*/		*/
ENTRY(copyout)		ENTRY(copyout_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax		movq PCPU(CURPCB),%rax
movq $copyout_fault,PCB_ONFAULT(%rax)		movq $copyout_fault,PCB_ONFAULT(%rax)
testq %rdx,%rdx /* anything to do? */		testq %rdx,%rdx /* anything to do? */
jz done_copyout		jz done_copyout

/*		/*
* Check explicitly for non-user addresses. This check is essential		* Check explicitly for non-user addresses. This check is essential
Show All 23 Lines	*/
shrq $3,%rcx		shrq $3,%rcx
rep		rep
movsq		movsq
movb %dl,%cl		movb %dl,%cl
andb $7,%cl		andb $7,%cl
rep		rep
movsb		movsb

		jmp done_copyout
		END(copyout_nosmap)

		ENTRY(copyout_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rax
		/* Trap entry clears PSL.AC */
		movq $copyout_fault,PCB_ONFAULT(%rax)
		testq %rdx,%rdx /* anything to do? */
		jz done_copyout

		/*
		* Check explicitly for non-user addresses. If 486 write protection
		jhbUnsubmitted Not Done Inline Actions I wonder if this comment is a bit stale? It doesn't seem like any pre-486 CPUs will ever support SMAP. OTOH, I'm not quite sure what is even meant by "486 write protection". jhb: I wonder if this comment is a bit stale? It doesn't seem like any pre-486 CPUs will ever…
		kibAuthorUnsubmitted Not Done Inline Actions As discussed, 'the protection' means CR0.WP=1. In fact this comment is wrong in more fundamental way. The lack of CoW when user tricked copyout to copy into KVA is the least of the problems. I rewrote it there and for i386. If CR0,WP=0, CoW would not work anyway, checked or not. kib: As discussed, 'the protection' means CR0.WP=1. In fact this comment is wrong in more…
		* is being used, this check is essential because we are in kernel
		* mode so the h/w does not provide any protection against writing
		* kernel addresses.
		*/

		/*
		* First, prevent address wrapping.
		*/
		movq %rsi,%rax
		addq %rdx,%rax
		jc copyout_fault
		/*
		* XXX STOP USING VM_MAXUSER_ADDRESS.
		* It is an end address, not a max, so every time it is used correctly it
		* looks like there is an off by one error, and of course it caused an off
		* by one error in several places.
		*/
		movq $VM_MAXUSER_ADDRESS,%rcx
		cmpq %rcx,%rax
		ja copyout_fault

		xchgq %rdi,%rsi
		/* bcopy(%rsi, %rdi, %rdx) */
		movq %rdx,%rcx

		shrq $3,%rcx
		cld
		stac
		rep
		movsq
		movb %dl,%cl
		andb $7,%cl
		rep
		movsb
		clac

done_copyout:		done_copyout:
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rdx		movq PCPU(CURPCB),%rdx
movq %rax,PCB_ONFAULT(%rdx)		movq %rax,PCB_ONFAULT(%rdx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret

ALIGN_TEXT		ALIGN_TEXT
copyout_fault:		copyout_fault:
movq PCPU(CURPCB),%rdx		movq PCPU(CURPCB),%rdx
movq $0,PCB_ONFAULT(%rdx)		movq $0,PCB_ONFAULT(%rdx)
movq $EFAULT,%rax		movq $EFAULT,%rax
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(copyout)		END(copyout)

/*		/*
* copyin(from_user, to_kernel, len)		* copyin(from_user, to_kernel, len)
* %rdi, %rsi, %rdx		* %rdi, %rsi, %rdx
*/		*/
ENTRY(copyin)		ENTRY(copyin_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rax		movq PCPU(CURPCB),%rax
movq $copyin_fault,PCB_ONFAULT(%rax)		movq $copyin_fault,PCB_ONFAULT(%rax)
testq %rdx,%rdx /* anything to do? */		testq %rdx,%rdx /* anything to do? */
jz done_copyin		jz done_copyin

/*		/*
* make sure address is valid		* make sure address is valid
*/		*/
movq %rdi,%rax		movq %rdi,%rax
addq %rdx,%rax		addq %rdx,%rax
jc copyin_fault		jc copyin_fault
movq $VM_MAXUSER_ADDRESS,%rcx		movq $VM_MAXUSER_ADDRESS,%rcx
cmpq %rcx,%rax		cmpq %rcx,%rax
ja copyin_fault		ja copyin_fault

xchgq %rdi,%rsi		xchgq %rdi,%rsi
movq %rdx,%rcx		movq %rdx,%rcx
movb %cl,%al		movb %cl,%al
shrq $3,%rcx /* copy longword-wise */		shrq $3,%rcx /* copy longword-wise */
		cld
		jhbUnsubmitted Not Done Inline Actions (I know this is old code, but I wonder if this 'cld' is even needed anymore since 'cld' is now part of the ABI?) jhb: (I know this is old code, but I wonder if this 'cld' is even needed anymore since 'cld' is now…
rep		rep
movsq		movsq
movb %al,%cl		movb %al,%cl
andb $7,%cl /* copy remaining bytes */		andb $7,%cl /* copy remaining bytes */
rep		rep
movsb		movsb

		jmp done_copyin
		END(copyin_nosmap)

		ENTRY(copyin_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rax
		movq $copyin_fault,PCB_ONFAULT(%rax)
		testq %rdx,%rdx /* anything to do? */
		jz done_copyin

		/*
		* make sure address is valid
		*/
		movq %rdi,%rax
		addq %rdx,%rax
		jc copyin_fault
		movq $VM_MAXUSER_ADDRESS,%rcx
		cmpq %rcx,%rax
		ja copyin_fault

		xchgq %rdi,%rsi
		movq %rdx,%rcx
		movb %cl,%al
		shrq $3,%rcx /* copy longword-wise */
		stac
		rep
		movsq
		movb %al,%cl
		andb $7,%cl /* copy remaining bytes */
		rep
		movsb
		clac

done_copyin:		done_copyin:
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rdx		movq PCPU(CURPCB),%rdx
movq %rax,PCB_ONFAULT(%rdx)		movq %rax,PCB_ONFAULT(%rdx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
		END(copyin_smap)

ALIGN_TEXT		ALIGN_TEXT
copyin_fault:		copyin_fault:
movq PCPU(CURPCB),%rdx		movq PCPU(CURPCB),%rdx
movq $0,PCB_ONFAULT(%rdx)		movq $0,PCB_ONFAULT(%rdx)
movq $EFAULT,%rax		movq $EFAULT,%rax
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(copyin)

/*		/*
* casueword32. Compare and set user integer. Returns -1 on fault,		* casueword32. Compare and set user integer. Returns -1 on fault,
* 0 if access was successful. Old value is written to *oldp.		* 0 if access was successful. Old value is written to *oldp.
* dst = %rdi, old = %esi, oldp = %rdx, new = %ecx		* dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
*/		*/
ENTRY(casueword32)		ENTRY(casueword32_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8		movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)		movq $fusufault,PCB_ONFAULT(%r8)

movq $VM_MAXUSER_ADDRESS-4,%rax		movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */		cmpq %rax,%rdi /* verify address is valid */
ja fusufault		ja fusufault

Show All 15 Lines	#endif

/*		/*
* Access the oldp after the pcb_onfault is cleared, to correctly		* Access the oldp after the pcb_onfault is cleared, to correctly
* catch corrupted pointer.		* catch corrupted pointer.
*/		*/
movl %esi,(%rdx) /* oldp = %rdx */		movl %esi,(%rdx) /* oldp = %rdx */
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(casueword32)		END(casueword32_nosmap)

		ENTRY(casueword32_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%r8
		movq $fusufault,PCB_ONFAULT(%r8)

		movq $VM_MAXUSER_ADDRESS-4,%rax
		cmpq %rax,%rdi /* verify address is valid */
		ja fusufault

		movl %esi,%eax /* old */
		stac
		#ifdef SMP
		lock
		#endif
		cmpxchgl %ecx,(%rdi) /* new = %ecx */
		clac

/*		/*
		* The old value is in %eax. If the store succeeded it will be the
		* value we expected (old) from before the store, otherwise it will
		* be the current value. Save %eax into %esi to prepare the return
		* value.
		*/
		movl %eax,%esi
		xorl %eax,%eax
		movq %rax,PCB_ONFAULT(%r8)

		/*
		* Access the oldp after the pcb_onfault is cleared, to correctly
		* catch corrupted pointer.
		*/
		movl %esi,(%rdx) /* oldp = %rdx */
		POP_FRAME_POINTER
		ret
		END(casueword32_smap)

		/*
* casueword. Compare and set user long. Returns -1 on fault,		* casueword. Compare and set user long. Returns -1 on fault,
* 0 if access was successful. Old value is written to *oldp.		* 0 if access was successful. Old value is written to *oldp.
* dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx		* dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
*/		*/
ENTRY(casueword)		ENTRY(casueword_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%r8		movq PCPU(CURPCB),%r8
movq $fusufault,PCB_ONFAULT(%r8)		movq $fusufault,PCB_ONFAULT(%r8)

movq $VM_MAXUSER_ADDRESS-4,%rax		movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */		cmpq %rax,%rdi /* verify address is valid */
ja fusufault		ja fusufault

Show All 9 Lines	#endif
* be the current value.		* be the current value.
*/		*/
movq %rax,%rsi		movq %rax,%rsi
xorl %eax,%eax		xorl %eax,%eax
movq %rax,PCB_ONFAULT(%r8)		movq %rax,PCB_ONFAULT(%r8)
movq %rsi,(%rdx)		movq %rsi,(%rdx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(casueword)		END(casueword_nosmap)

		ENTRY(casueword_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%r8
		movq $fusufault,PCB_ONFAULT(%r8)

		movq $VM_MAXUSER_ADDRESS-4,%rax
		cmpq %rax,%rdi /* verify address is valid */
		ja fusufault

		movq %rsi,%rax /* old */
		stac
		#ifdef SMP
		lock
		#endif
		cmpxchgq %rcx,(%rdi) /* new = %rcx */
		clac

/*		/*
		* The old value is in %rax. If the store succeeded it will be the
		* value we expected (old) from before the store, otherwise it will
		* be the current value.
		*/
		movq %rax,%rsi
		xorl %eax,%eax
		movq %rax,PCB_ONFAULT(%r8)
		movq %rsi,(%rdx)
		POP_FRAME_POINTER
		ret
		END(casueword_smap)

		/*
* Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit		* Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
* byte from user memory.		* byte from user memory.
* addr = %rdi, valp = %rsi		* addr = %rdi, valp = %rsi
*/		*/

ALTENTRY(fueword64)		ENTRY(fueword_nosmap)
ENTRY(fueword)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-8,%rax		movq $VM_MAXUSER_ADDRESS-8,%rax
cmpq %rax,%rdi /* verify address is valid */		cmpq %rax,%rdi /* verify address is valid */
ja fusufault		ja fusufault

xorl %eax,%eax		xorl %eax,%eax
movq (%rdi),%r11		movq (%rdi),%r11
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
movq %r11,(%rsi)		movq %r11,(%rsi)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(fueword64)		END(fueword64_nosmap)
END(fueword)

ENTRY(fueword32)		ENTRY(fueword_smap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-8,%rax
		cmpq %rax,%rdi /* verify address is valid */
		ja fusufault

		xorl %eax,%eax
		stac
		movq (%rdi),%r11
		clac
		movq %rax,PCB_ONFAULT(%rcx)
		movq %r11,(%rsi)
		POP_FRAME_POINTER
		ret
		END(fueword64_smap)

		ENTRY(fueword32_nosmap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-4,%rax		movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address is valid */		cmpq %rax,%rdi /* verify address is valid */
ja fusufault		ja fusufault

xorl %eax,%eax		xorl %eax,%eax
movl (%rdi),%r11d		movl (%rdi),%r11d
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
movl %r11d,(%rsi)		movl %r11d,(%rsi)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(fueword32)		END(fueword32_nosmap)

		ENTRY(fueword32_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-4,%rax
		cmpq %rax,%rdi /* verify address is valid */
		ja fusufault

		xorl %eax,%eax
		stac
		movl (%rdi),%r11d
		clac
		movq %rax,PCB_ONFAULT(%rcx)
		movl %r11d,(%rsi)
		POP_FRAME_POINTER
		ret
		END(fueword32_smap)

/*		/*
* fuswintr() and suswintr() are specialized variants of fuword16() and		* fuswintr() and suswintr() are specialized variants of fuword16() and
* suword16(), respectively. They are called from the profiling code,		* suword16(), respectively. They are called from the profiling code,
* potentially at interrupt time. If they fail, that's okay; good things		* potentially at interrupt time. If they fail, that's okay; good things
* will happen later. They always fail for now, until the trap code is		* will happen later. They always fail for now, until the trap code is
* able to deal with this.		* able to deal with this.
*/		*/
ALTENTRY(suswintr)		ALTENTRY(suswintr)
ENTRY(fuswintr)		ENTRY(fuswintr)
movq $-1,%rax		movq $-1,%rax
ret		ret
END(suswintr)		END(suswintr)
END(fuswintr)		END(fuswintr)

ENTRY(fuword16)		ENTRY(fuword16_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-2,%rax		movq $VM_MAXUSER_ADDRESS-2,%rax
cmpq %rax,%rdi		cmpq %rax,%rdi
ja fusufault		ja fusufault

movzwl (%rdi),%eax		movzwl (%rdi),%eax
movq $0,PCB_ONFAULT(%rcx)		movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(fuword16)		END(fuword16_nosmap)

ENTRY(fubyte)		ENTRY(fuword16_smap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-2,%rax
		cmpq %rax,%rdi
		ja fusufault

		stac
		movzwl (%rdi),%eax
		clac
		movq $0,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(fuword16_smap)

		ENTRY(fubyte_nosmap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-1,%rax		movq $VM_MAXUSER_ADDRESS-1,%rax
cmpq %rax,%rdi		cmpq %rax,%rdi
ja fusufault		ja fusufault

movzbl (%rdi),%eax		movzbl (%rdi),%eax
movq $0,PCB_ONFAULT(%rcx)		movq $0,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(fubyte)		END(fubyte_nosmap)

		ENTRY(fubyte_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-1,%rax
		cmpq %rax,%rdi
		ja fusufault

		stac
		movzbl (%rdi),%eax
		clac
		movq $0,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(fubyte_smap)

ALIGN_TEXT		ALIGN_TEXT
		/* Fault entry clears PSL.AC */
fusufault:		fusufault:
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
xorl %eax,%eax		xorl %eax,%eax
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
decq %rax		decq %rax
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret

/*		/*
* Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to		* Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
* user memory.		* user memory.
* addr = %rdi, value = %rsi		* addr = %rdi, value = %rsi
*/		*/
ALTENTRY(suword64)		ENTRY(suword_nosmap)
ENTRY(suword)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-8,%rax		movq $VM_MAXUSER_ADDRESS-8,%rax
cmpq %rax,%rdi /* verify address validity */		cmpq %rax,%rdi /* verify address validity */
ja fusufault		ja fusufault

movq %rsi,(%rdi)		movq %rsi,(%rdi)
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(suword64)		END(suword_nosmap)
END(suword)

ENTRY(suword32)		ENTRY(suword_smap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-8,%rax
		cmpq %rax,%rdi /* verify address validity */
		ja fusufault

		stac
		movq %rsi,(%rdi)
		clac
		xorl %eax,%eax
		movq PCPU(CURPCB),%rcx
		movq %rax,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(suword_smap)

		ENTRY(suword32_nosmap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-4,%rax		movq $VM_MAXUSER_ADDRESS-4,%rax
cmpq %rax,%rdi /* verify address validity */		cmpq %rax,%rdi /* verify address validity */
ja fusufault		ja fusufault

movl %esi,(%rdi)		movl %esi,(%rdi)
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(suword32)		END(suword32_nosmap)

ENTRY(suword16)		ENTRY(suword32_smap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-4,%rax
		cmpq %rax,%rdi /* verify address validity */
		ja fusufault

		stac
		movl %esi,(%rdi)
		clac
		xorl %eax,%eax
		movq PCPU(CURPCB),%rcx
		movq %rax,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(suword32_smap)

		ENTRY(suword16_nosmap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-2,%rax		movq $VM_MAXUSER_ADDRESS-2,%rax
cmpq %rax,%rdi /* verify address validity */		cmpq %rax,%rdi /* verify address validity */
ja fusufault		ja fusufault

movw %si,(%rdi)		movw %si,(%rdi)
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rcx /* restore trashed register */		movq PCPU(CURPCB),%rcx /* restore trashed register */
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(suword16)		END(suword16_nosmap)

ENTRY(subyte)		ENTRY(suword16_smap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $fusufault,PCB_ONFAULT(%rcx)		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-2,%rax
		cmpq %rax,%rdi /* verify address validity */
		ja fusufault

		stac
		movw %si,(%rdi)
		clac
		xorl %eax,%eax
		movq PCPU(CURPCB),%rcx /* restore trashed register */
		movq %rax,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(suword16_smap)

		ENTRY(subyte_nosmap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS-1,%rax		movq $VM_MAXUSER_ADDRESS-1,%rax
cmpq %rax,%rdi /* verify address validity */		cmpq %rax,%rdi /* verify address validity */
ja fusufault		ja fusufault

movl %esi,%eax		movl %esi,%eax
movb %al,(%rdi)		movb %al,(%rdi)
xorl %eax,%eax		xorl %eax,%eax
movq PCPU(CURPCB),%rcx /* restore trashed register */		movq PCPU(CURPCB),%rcx /* restore trashed register */
movq %rax,PCB_ONFAULT(%rcx)		movq %rax,PCB_ONFAULT(%rcx)
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(subyte)		END(subyte_nosmap)

		ENTRY(subyte_smap)
		PUSH_FRAME_POINTER
		movq PCPU(CURPCB),%rcx
		movq $fusufault,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS-1,%rax
		cmpq %rax,%rdi /* verify address validity */
		ja fusufault

		movl %esi,%eax
		stac
		movb %al,(%rdi)
		clac
		xorl %eax,%eax
		movq PCPU(CURPCB),%rcx /* restore trashed register */
		movq %rax,PCB_ONFAULT(%rcx)
		POP_FRAME_POINTER
		ret
		END(subyte_smap)

/*		/*
* copyinstr(from, to, maxlen, int *lencopied)		* copyinstr(from, to, maxlen, int *lencopied)
* %rdi, %rsi, %rdx, %rcx		* %rdi, %rsi, %rdx, %rcx
*		*
* copy a string from 'from' to 'to', stop when a 0 character is reached.		* copy a string from 'from' to 'to', stop when a 0 character is reached.
* return ENAMETOOLONG if string is longer than maxlen, and		* return ENAMETOOLONG if string is longer than maxlen, and
* EFAULT on protection violations. If lencopied is non-zero,		* EFAULT on protection violations. If lencopied is non-zero,
* return the actual length in *lencopied.		* return the actual length in *lencopied.
*/		*/
ENTRY(copyinstr)		ENTRY(copyinstr_nosmap)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq %rdx,%r8 /* %r8 = maxlen */		movq %rdx,%r8 /* %r8 = maxlen */
movq %rcx,%r9 /* %r9 = len /		movq %rcx,%r9 /* %r9 = len /
xchgq %rdi,%rsi /* %rdi = from, %rsi = to */		xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $cpystrflt,PCB_ONFAULT(%rcx)		movq $cpystrflt,PCB_ONFAULT(%rcx)

movq $VM_MAXUSER_ADDRESS,%rax		movq $VM_MAXUSER_ADDRESS,%rax

/* make sure 'from' is within bounds */		/* make sure 'from' is within bounds */
subq %rsi,%rax		subq %rsi,%rax
jbe cpystrflt		jbe cpystrflt

/* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */		/* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
cmpq %rdx,%rax		cmpq %rdx,%rax
jae 1f		jae 1f
movq %rax,%rdx		movq %rax,%rdx
movq %rax,%r8		movq %rax,%r8
1:		1:
incq %rdx		incq %rdx
		cld

2:		2:
decq %rdx		decq %rdx
jz 3f		jz copyinstr_toolong

lodsb		lodsb
stosb		stosb
orb %al,%al		orb %al,%al
jnz 2b		jnz 2b

		jmp copyinstr_succ
		END(copyinstr_nosmap)

		ENTRY(copyinstr_smap)
		PUSH_FRAME_POINTER
		movq %rdx,%r8 /* %r8 = maxlen */
		movq %rcx,%r9 /* %r9 = len /
		xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
		movq PCPU(CURPCB),%rcx
		movq $cpystrflt,PCB_ONFAULT(%rcx)

		movq $VM_MAXUSER_ADDRESS,%rax

		/* make sure 'from' is within bounds */
		subq %rsi,%rax
		jbe cpystrflt

		/* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
		cmpq %rdx,%rax
		jae 1f
		movq %rax,%rdx
		movq %rax,%r8
		1:
		incq %rdx

		2:
		decq %rdx
		jz copyinstr_succ

		stac
		lodsb
		stosb
		clac
		orb %al,%al
		jnz 2b

		copyinstr_succ:
/* Success -- 0 byte reached */		/* Success -- 0 byte reached */
decq %rdx		decq %rdx
xorl %eax,%eax		xorl %eax,%eax
jmp cpystrflt_x		jmp cpystrflt_x
3:		copyinstr_toolong:
/* rdx is zero - return ENAMETOOLONG or EFAULT */		/* rdx is zero - return ENAMETOOLONG or EFAULT */
movq $VM_MAXUSER_ADDRESS,%rax		movq $VM_MAXUSER_ADDRESS,%rax
cmpq %rax,%rsi		cmpq %rax,%rsi
jae cpystrflt		jae cpystrflt
4:
movq $ENAMETOOLONG,%rax		movq $ENAMETOOLONG,%rax
jmp cpystrflt_x		jmp cpystrflt_x

		/* Fault entry clears PSL.AC */
cpystrflt:		cpystrflt:
movq $EFAULT,%rax		movq $EFAULT,%rax

cpystrflt_x:		cpystrflt_x:
/* set lencopied and return %eax /		/* set lencopied and return %eax /
movq PCPU(CURPCB),%rcx		movq PCPU(CURPCB),%rcx
movq $0,PCB_ONFAULT(%rcx)		movq $0,PCB_ONFAULT(%rcx)

testq %r9,%r9		testq %r9,%r9
jz 1f		jz 1f
subq %rdx,%r8		subq %rdx,%r8
movq %r8,(%r9)		movq %r8,(%r9)
1:		1:
POP_FRAME_POINTER		POP_FRAME_POINTER
ret		ret
END(copyinstr)		END(copyinstr_smap)

/*		/*
* copystr(from, to, maxlen, int *lencopied)		* copystr(from, to, maxlen, int *lencopied)
* %rdi, %rsi, %rdx, %rcx		* %rdi, %rsi, %rdx, %rcx
*/		*/
ENTRY(copystr)		ENTRY(copystr)
PUSH_FRAME_POINTER		PUSH_FRAME_POINTER
movq %rdx,%r8 /* %r8 = maxlen */		movq %rdx,%r8 /* %r8 = maxlen */
▲ Show 20 Lines • Show All 141 Lines • Show Last 20 Lines