Page MenuHomeFreeBSD
Differential D17124

ERMS-based memmove, memcpy and memset
ClosedPublic
Actions

Authored by mjg on Sep 11 2018, 11:15 PM.
Tags
None
Referenced Files
Unknown Object (File)
Wed, Mar 12, 7:03 PM
Unknown Object (File)
Sun, Mar 9, 1:32 AM
Unknown Object (File)
Feb 16 2025, 11:35 PM
Unknown Object (File)
Jan 9 2025, 12:04 AM
Unknown Object (File)
Jan 7 2025, 11:33 PM
Unknown Object (File)
Dec 13 2024, 11:33 AM
Unknown Object (File)
Dec 4 2024, 9:32 AM
Unknown Object (File)
Nov 4 2024, 1:15 AM
View All 84 Files
Subscribers
imp

Details

Reviewers
kib
Commits
rS338645: amd64: implement ERMS-based memmove, memcpy and memset
Summary

Tested with and without invariants.

I don't know where to put the resolving code, machdep.c seemed like the least objectionable place.

Note that memmove is kind of hairy and is up for further rework.

Diff Detail

Lint
Lint Skipped
Unit
Tests Skipped

Event Timeline

mjg created this revision.Sep 11 2018, 11:15 PM
Herald added a subscriber: imp. · View Herald TranscriptSep 11 2018, 11:15 PM
Harbormaster completed remote builds in B19533: Diff 47932.Sep 11 2018, 11:15 PM
kib added a comment.Sep 12 2018, 2:15 PM

Please upload patch with context. svn diff -x -U99999

sys/amd64/amd64/support.S
1269

Move this right after _std. Also, I prefer to have each _erms variant to be located right after its _std counterpart.

1293

What is the purpose of the comment ?

mjg updated this revision to Diff 47960.Sep 12 2018, 5:50 PM
  • remove the comment copied over from original memmove
  • include full-context diff
mjg updated this revision to Diff 47997.Sep 13 2018, 1:47 PM
  • put erms funcs net to their standard counterparts
kib accepted this revision.Sep 13 2018, 2:35 PM
This revision is now accepted and ready to land.Sep 13 2018, 2:35 PM
Closed by commit rS338645: amd64: implement ERMS-based memmove, memcpy and memset (authored by mjg). · Explain WhySep 13 2018, 2:54 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathSize
sys/
amd64/
amd64/
32 lines
82 lines

Diff 47960

View Options

sys/amd64/amd64/machdep.c

Show First 20 Lines • Show All 125 Lines • ▼ Show 20 Lines
#include <machine/pcb.h> #include <machine/pcb.h>
#include <machine/proc.h> #include <machine/proc.h>
#include <machine/reg.h> #include <machine/reg.h>
#include <machine/sigframe.h> #include <machine/sigframe.h>
#include <machine/specialreg.h> #include <machine/specialreg.h>
#include <machine/trap.h> #include <machine/trap.h>
#include <machine/tss.h> #include <machine/tss.h>
#include <x86/ucode.h> #include <x86/ucode.h>
#include <x86/ifunc.h>
#ifdef SMP #ifdef SMP
#include <machine/smp.h> #include <machine/smp.h>
#endif #endif
#ifdef FDT #ifdef FDT
#include <x86/fdt.h> #include <x86/fdt.h>
#endif #endif
#ifdef DEV_ATPIC #ifdef DEV_ATPIC
▲ Show 20 Lines • Show All 2,514 Lines • ▼ Show 20 Lines
void void
outb_(u_short port, u_char data) outb_(u_short port, u_char data)
{ {
outb(port, data); outb(port, data);
} }
#endif /* KDB */ #endif /* KDB */
#undef memset
#undef memmove
#undef memcpy
void *memset_std(void *buf, int c, size_t len);
void *memset_erms(void *buf, int c, size_t len);
DEFINE_IFUNC(, void *, memset, (void *, int, size_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memset_erms : memset_std);
}
void *memmove_std(void * _Nonnull dst, const void * _Nonnull src, size_t len);
void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src, size_t len);
DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull, size_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memmove_erms : memmove_std);
}
void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src, size_t len);
void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src, size_t len);
DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull, size_t), static)
{
return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
memcpy_erms : memcpy_std);
}
View Options

sys/amd64/amd64/support.S

Show First 20 Lines • Show All 90 Lines • ▼ Show 20 Lines
END(sse2_pagezero) END(sse2_pagezero)
/* /*
* memmove(dst, src, cnt) * memmove(dst, src, cnt)
* rdi, rsi, rdx * rdi, rsi, rdx
* Adapted from bcopy written by: * Adapted from bcopy written by:
* ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
*/ */
ENTRY(memmove) ENTRY(memmove_std)
PUSH_FRAME_POINTER PUSH_FRAME_POINTER
movq %rdi,%r9 movq %rdi,%r9
movq %rdx,%rcx movq %rdx,%rcx
movq %rdi,%rax movq %rdi,%rax
subq %rsi,%rax subq %rsi,%rax
cmpq %rcx,%rax /* overlapping && src < dst? */ cmpq %rcx,%rax /* overlapping && src < dst? */
jb 1f jb 1f
Show All 29 Lines1:
subq $7,%rsi subq $7,%rsi
subq $7,%rdi subq $7,%rdi
rep rep
movsq movsq
cld cld
movq %r9,%rax movq %r9,%rax
POP_FRAME_POINTER POP_FRAME_POINTER
ret ret
END(memmove) END(memmove_std)
/* /*
* memcpy(dst, src, len) * memcpy(dst, src, len)
* rdi, rsi, rdx * rdi, rsi, rdx
* *
* Note: memcpy does not support overlapping copies * Note: memcpy does not support overlapping copies
*/ */
ENTRY(memcpy) ENTRY(memcpy_std)
PUSH_FRAME_POINTER PUSH_FRAME_POINTER
movq %rdi,%rax movq %rdi,%rax
movq %rdx,%rcx movq %rdx,%rcx
shrq $3,%rcx /* copy by 64-bit words */ shrq $3,%rcx /* copy by 64-bit words */
rep rep
movsq movsq
movq %rdx,%rcx movq %rdx,%rcx
andq $7,%rcx /* any bytes left? */ andq $7,%rcx /* any bytes left? */
jne 1f jne 1f
POP_FRAME_POINTER POP_FRAME_POINTER
ret ret
1: 1:
rep rep
movsb movsb
POP_FRAME_POINTER POP_FRAME_POINTER
ret ret
END(memcpy) END(memcpy_std)
/* /*
* memset(dst, c, len) * memset(dst, c, len)
* rdi, rsi, rdx * rdi, rsi, rdx
*/ */
ENTRY(memset) ENTRY(memset_std)
PUSH_FRAME_POINTER PUSH_FRAME_POINTER
movq %rdi,%r9 movq %rdi,%r9
movq %rdx,%rcx movq %rdx,%rcx
movzbq %sil,%r8 movzbq %sil,%r8
movabs $0x0101010101010101,%rax movabs $0x0101010101010101,%rax
imulq %r8,%rax imulq %r8,%rax
shrq $3,%rcx shrq $3,%rcx
rep rep
stosq stosq
movq %rdx,%rcx movq %rdx,%rcx
andq $7,%rcx andq $7,%rcx
jne 1f jne 1f
movq %r9,%rax movq %r9,%rax
POP_FRAME_POINTER POP_FRAME_POINTER
ret ret
1: 1:
rep rep
stosb stosb
movq %r9,%rax movq %r9,%rax
POP_FRAME_POINTER POP_FRAME_POINTER
ret ret
END(memset) END(memset_std)
/* fillw(pat, base, cnt) */ /* fillw(pat, base, cnt) */
/* %rdi,%rsi, %rdx */ /* %rdi,%rsi, %rdx */
ENTRY(fillw) ENTRY(fillw)
PUSH_FRAME_POINTER PUSH_FRAME_POINTER
movq %rdi,%rax movq %rdi,%rax
movq %rsi,%rdi movq %rsi,%rdi
movq %rdx,%rcx movq %rdx,%rcx
▲ Show 20 Lines • Show All 1,053 Lines • ▼ Show 20 Lines1: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
/* pass 2: Read each cache line. */ /* pass 2: Read each cache line. */
2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 2: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
addq $64, %rcx addq $64, %rcx
jne 2b jne 2b
lfence lfence
ret ret
#undef L1D_FLUSH_SIZE #undef L1D_FLUSH_SIZE
END(flush_l1d_sw) END(flush_l1d_sw)
/* Variants utilizing Enhanced REP MOVSB/STOSB, commonly found on Intel CPUs. */
kibUnsubmitted
Not Done
Inline Actions

Move this right after _std. Also, I prefer to have each _erms variant to be located right after its _std counterpart.

kib: Move this right after _std. Also, I prefer to have each _erms variant to be located right…
/*
* memmove(dst, src, cnt)
* rdi, rsi, rdx
* Adapted from bcopy written by:
* ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
*/
ENTRY(memmove_erms)
PUSH_FRAME_POINTER
movq %rdi,%r9
movq %rdx,%rcx
movq %rdi,%rax
subq %rsi,%rax
cmpq %rcx,%rax /* overlapping && src < dst? */
jb 1f
rep
movsb
movq %r9,%rax
POP_FRAME_POINTER
ret
1:
kibUnsubmitted
Not Done
Inline Actions

What is the purpose of the comment ?

kib: What is the purpose of the comment ?
addq %rcx,%rdi /* copy backwards */
addq %rcx,%rsi
decq %rdi
decq %rsi
std
rep
movsb
cld
movq %r9,%rax
POP_FRAME_POINTER
ret
END(memmove_erms)
/*
* memcpy(dst, src, len)
* rdi, rsi, rdx
*
* Note: memcpy does not support overlapping copies
*/
ENTRY(memcpy_erms)
PUSH_FRAME_POINTER
movq %rdi,%rax
movq %rdx,%rcx
rep
movsb
POP_FRAME_POINTER
ret
END(memcpy)
/*
* memset(dst, c, len)
* rdi, rsi, rdx
*/
ENTRY(memset_erms)
PUSH_FRAME_POINTER
movq %rdi,%r9
movq %rdx,%rcx
movb %sil,%al
rep
stosb
movq %r9,%rax
POP_FRAME_POINTER
ret
END(memset_erms)