Index: head/sys/amd64/amd64/elf_machdep.c =================================================================== --- head/sys/amd64/amd64/elf_machdep.c +++ head/sys/amd64/amd64/elf_machdep.c @@ -186,7 +186,7 @@ /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, elf_lookup_fn lookup) + int type, bool late_ifunc, elf_lookup_fn lookup) { Elf64_Addr *where, val; Elf32_Addr *where32, val32; @@ -226,6 +226,13 @@ panic("unknown reloc type %d\n", type); } + if (late_ifunc) { + KASSERT(type == ELF_RELOC_RELA, + ("Only RELA ifunc relocations are supported")); + if (rtype != R_X86_64_IRELATIVE) + return (0); + } + switch (rtype) { case R_X86_64_NONE: /* none */ break; @@ -305,7 +312,7 @@ elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, false, lookup)); } int @@ -313,7 +320,15 @@ int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, false, lookup)); +} + +int +elf_reloc_late(linker_file_t lf, Elf_Addr relocbase, const void *data, + int type, elf_lookup_fn lookup) +{ + + return (elf_reloc_internal(lf, relocbase, data, type, true, lookup)); } int Index: head/sys/amd64/amd64/machdep.c =================================================================== --- head/sys/amd64/amd64/machdep.c +++ head/sys/amd64/amd64/machdep.c @@ -320,6 +320,13 @@ cpu_setregs(); } +static void +late_ifunc_resolve(void *dummy __unused) +{ + link_elf_late_ireloc(); +} +SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); + /* * Send an interrupt to process. * Index: head/sys/arm64/arm64/elf_machdep.c =================================================================== --- head/sys/arm64/arm64/elf_machdep.c +++ head/sys/arm64/arm64/elf_machdep.c @@ -143,8 +143,10 @@ */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, int local, elf_lookup_fn lookup) + int type, int flags, elf_lookup_fn lookup) { +#define ARM64_ELF_RELOC_LOCAL (1 << 0) +#define ARM64_ELF_RELOC_LATE_IFUNC (1 << 1) Elf_Addr *where, addr, addend, val; Elf_Word rtype, symidx; const Elf_Rel *rel; @@ -170,7 +172,14 @@ panic("unknown reloc type %d\n", type); } - if (local) { + if ((flags & ARM64_ELF_RELOC_LATE_IFUNC) != 0) { + KASSERT(type == ELF_RELOC_RELA, + ("Only RELA ifunc relocations are supported")); + if (rtype != R_AARCH64_IRELATIVE) + return (0); + } + + if ((flags & ARM64_ELF_RELOC_LOCAL) != 0) { if (rtype == R_AARCH64_RELATIVE) *where = elf_relocaddr(lf, relocbase + addend); return (0); @@ -229,7 +238,8 @@ int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup)); + return (elf_reloc_internal(lf, relocbase, data, type, + ARM64_ELF_RELOC_LOCAL, lookup)); } /* Process one elf relocation with addend. */ @@ -239,6 +249,15 @@ { return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup)); +} + +int +elf_reloc_late(linker_file_t lf, Elf_Addr relocbase, const void *data, + int type, elf_lookup_fn lookup) +{ + + return (elf_reloc_internal(lf, relocbase, data, type, + ARM64_ELF_RELOC_LATE_IFUNC, lookup)); } int Index: head/sys/arm64/arm64/machdep.c =================================================================== --- head/sys/arm64/arm64/machdep.c +++ head/sys/arm64/arm64/machdep.c @@ -181,6 +181,13 @@ SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); +static void +late_ifunc_resolve(void *dummy __unused) +{ + link_elf_late_ireloc(); +} +SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); + int cpu_idle_wakeup(int cpu) { Index: head/sys/kern/link_elf.c =================================================================== --- head/sys/kern/link_elf.c +++ head/sys/kern/link_elf.c @@ -1925,4 +1925,18 @@ link_elf_preload_parse_symbols(ef); relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); } + +#if defined(__aarch64__) || defined(__amd64__) +void +link_elf_late_ireloc(void) +{ + elf_file_t ef; + + KASSERT(linker_kernel_file != NULL, + ("link_elf_late_ireloc: No kernel linker file found")); + ef = (elf_file_t)linker_kernel_file; + + relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); +} +#endif #endif Index: head/sys/sys/linker.h =================================================================== --- head/sys/sys/linker.h +++ head/sys/sys/linker.h @@ -289,6 +289,12 @@ const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx); void link_elf_ireloc(caddr_t kmdp); +#if defined(__aarch64__) || defined(__amd64__) +int elf_reloc_late(linker_file_t _lf, Elf_Addr base, const void *_rel, + int _type, elf_lookup_fn _lu); +void link_elf_late_ireloc(void); +#endif + typedef struct linker_ctf { const uint8_t *ctftab; /* Decompressed CTF data. */ int ctfcnt; /* Number of CTF data bytes. */