Index: head/sys/amd64/amd64/elf_machdep.c =================================================================== --- head/sys/amd64/amd64/elf_machdep.c +++ head/sys/amd64/amd64/elf_machdep.c @@ -175,13 +175,17 @@ *off = len; } -#define ERI_LOCAL 0x0001 -#define ERI_ONLYIFUNC 0x0002 +bool +elf_is_ifunc_reloc(Elf_Size r_info) +{ + return (ELF_R_TYPE(r_info) == R_X86_64_IRELATIVE); +} + /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, elf_lookup_fn lookup, int flags) + int type, elf_lookup_fn lookup) { Elf64_Addr *where, val; Elf32_Addr *where32, val32; @@ -221,9 +225,6 @@ panic("unknown reloc type %d\n", type); } - if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_X86_64_IRELATIVE)) - return (0); - switch (rtype) { case R_X86_64_NONE: /* none */ break; @@ -300,20 +301,11 @@ } int -elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, elf_lookup_fn lookup) -{ - - return (elf_reloc_internal(lf, relocbase, data, type, lookup, - ERI_ONLYIFUNC)); -} - -int elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, lookup, 0)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup)); } int @@ -321,8 +313,7 @@ int type, elf_lookup_fn lookup) { - return (elf_reloc_internal(lf, relocbase, data, type, lookup, - ERI_LOCAL)); + return (elf_reloc_internal(lf, relocbase, data, type, lookup)); } int Index: head/sys/arm/arm/elf_machdep.c =================================================================== --- head/sys/arm/arm/elf_machdep.c +++ head/sys/arm/arm/elf_machdep.c @@ -149,6 +149,13 @@ #endif } +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + /* * It is possible for the compiler to emit relocations for unaligned data. * We handle this situation with these inlines. Index: head/sys/arm64/arm64/elf_machdep.c =================================================================== --- head/sys/arm64/arm64/elf_machdep.c +++ head/sys/arm64/arm64/elf_machdep.c @@ -129,6 +129,13 @@ } +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, int local, elf_lookup_fn lookup) Index: head/sys/i386/i386/elf_machdep.c =================================================================== --- head/sys/i386/i386/elf_machdep.c +++ head/sys/i386/i386/elf_machdep.c @@ -159,8 +159,14 @@ *off = len; } +bool +elf_is_ifunc_reloc(Elf_Size r_info) +{ + + return (ELF_R_TYPE(r_info) == R_386_IRELATIVE); +} + #define ERI_LOCAL 0x0001 -#define ERI_ONLYIFUNC 0x0002 /* Process one elf relocation with addend. */ static int @@ -194,9 +200,6 @@ panic("unknown reloc type %d\n", type); } - if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_386_IRELATIVE)) - return (0); - if ((flags & ERI_LOCAL) != 0) { if (rtype == R_386_RELATIVE) { /* A + B */ addr = elf_relocaddr(lf, relocbase + addend); @@ -261,15 +264,6 @@ return -1; } return(0); -} - -int -elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, elf_lookup_fn lookup) -{ - - return (elf_reloc_internal(lf, relocbase, data, type, lookup, - ERI_ONLYIFUNC)); } int Index: head/sys/kern/link_elf.c =================================================================== --- head/sys/kern/link_elf.c +++ head/sys/kern/link_elf.c @@ -188,11 +188,13 @@ link_elf_methods, sizeof(struct elf_file) }; +typedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, + const void *data, int type, elf_lookup_fn lookup); + static int parse_dynamic(elf_file_t); static int relocate_file(elf_file_t); -static int relocate_file1(elf_file_t ef, int (*elf_reloc_func)( - linker_file_t lf, Elf_Addr relocbase, const void *data, - int type, elf_lookup_fn lookup)); +static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, + elf_reloc_fn reloc, bool ifuncs); static int link_elf_preload_parse_symbols(elf_file_t); static struct elf_set_head set_pcpu_list; @@ -1185,96 +1187,61 @@ } static int -relocate_file1(elf_file_t ef, int (*elf_reloc_func)(linker_file_t lf, - Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup)) +symbol_type(elf_file_t ef, Elf_Size r_info) { - const Elf_Rel *rellim; + const Elf_Sym *ref; + + if (ELF_R_SYM(r_info)) { + ref = ef->symtab + ELF_R_SYM(r_info); + return (ELF_ST_TYPE(ref->st_info)); + } + return (STT_NOTYPE); +} + +static int +relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, + bool ifuncs) +{ const Elf_Rel *rel; - const Elf_Rela *relalim; const Elf_Rela *rela; const char *symname; - /* Perform relocations without addend if there are any: */ - rel = ef->rel; - if (rel != NULL) { - rellim = (const Elf_Rel *) - ((const char *)ef->rel + ef->relsize); - while (rel < rellim) { - if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel, - ELF_RELOC_REL, elf_lookup)) { - symname = symbol_name(ef, rel->r_info); - printf("link_elf: symbol %s undefined\n", symname); - return (ENOENT); - } - rel++; - } - } +#define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ + for ((iter) = (tbl); (iter) != NULL && \ + (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ + if ((symbol_type(ef, (iter)->r_info) == \ + STT_GNU_IFUNC || \ + elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ + continue; \ + if (reloc(&ef->lf, (Elf_Addr)ef->address, \ + (iter), (type), lookup)) { \ + symname = symbol_name(ef, (iter)->r_info); \ + printf("link_elf: symbol %s undefined\n", \ + symname); \ + return (ENOENT); \ + } \ + } \ +} while (0) - /* Perform relocations with addend if there are any: */ - rela = ef->rela; - if (rela != NULL) { - relalim = (const Elf_Rela *) - ((const char *)ef->rela + ef->relasize); - while (rela < relalim) { - if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela, - ELF_RELOC_RELA, elf_lookup)) { - symname = symbol_name(ef, rela->r_info); - printf("link_elf: symbol %s undefined\n", - symname); - return (ENOENT); - } - rela++; - } - } + APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); + APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); + APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); + APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); - /* Perform PLT relocations without addend if there are any: */ - rel = ef->pltrel; - if (rel != NULL) { - rellim = (const Elf_Rel *) - ((const char *)ef->pltrel + ef->pltrelsize); - while (rel < rellim) { - if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel, - ELF_RELOC_REL, elf_lookup)) { - symname = symbol_name(ef, rel->r_info); - printf("link_elf: symbol %s undefined\n", - symname); - return (ENOENT); - } - rel++; - } - } +#undef APPLY_RELOCS - /* Perform relocations with addend if there are any: */ - rela = ef->pltrela; - if (rela != NULL) { - relalim = (const Elf_Rela *) - ((const char *)ef->pltrela + ef->pltrelasize); - while (rela < relalim) { - if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela, - ELF_RELOC_RELA, elf_lookup)) { - symname = symbol_name(ef, rela->r_info); - printf("link_elf: symbol %s undefined\n", - symname); - return (ENOENT); - } - rela++; - } - } - return (0); } static int relocate_file(elf_file_t ef) { - int e; + int error; - e = relocate_file1(ef, elf_reloc); -#if defined(__i386__) || defined(__amd64__) - if (e == 0) - e = relocate_file1(ef, elf_reloc_ifunc); -#endif - return (e); + error = relocate_file1(ef, elf_lookup, elf_reloc, false); + if (error == 0) + error = relocate_file1(ef, elf_lookup, elf_reloc, true); + return (error); } /* @@ -1298,7 +1265,7 @@ } static int -link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t* sym) +link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) { elf_file_t ef = (elf_file_t) lf; unsigned long symnum; @@ -1687,6 +1654,29 @@ } #if defined(__i386__) || defined(__amd64__) +/* + * Use this lookup routine when performing relocations early during boot. + * The generic lookup routine depends on kobj, which is not initialized + * at that point. + */ +static int +elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, + Elf_Addr *res) +{ + elf_file_t ef; + const Elf_Sym *symp; + caddr_t val; + + ef = (elf_file_t)lf; + symp = ef->symtab + symidx; + if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { + val = (caddr_t)ef->address + symp->st_value; + *res = ((Elf_Addr (*)(void))val)(); + return (0); + } + return (ENOENT); +} + void link_elf_ireloc(caddr_t kmdp) { @@ -1695,7 +1685,7 @@ volatile char *c; size_t i; - ef = &eff; + ef = &eff; /* Do not use bzero/memset before ireloc is done. */ for (c = (char *)ef, i = 0; i < sizeof(*ef); i++) @@ -1706,6 +1696,6 @@ parse_dynamic(ef); ef->address = 0; link_elf_preload_parse_symbols(ef); - relocate_file1(ef, elf_reloc_ifunc); + relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); } #endif Index: head/sys/kern/link_elf_obj.c =================================================================== --- head/sys/kern/link_elf_obj.c +++ head/sys/kern/link_elf_obj.c @@ -1521,15 +1521,10 @@ /* Only do local relocs */ if (ELF_ST_BIND(sym->st_info) != STB_LOCAL) continue; - if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC) == - ifuncs) + if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || + elf_is_ifunc_reloc(rel->r_info)) == ifuncs) elf_reloc_local(lf, base, rel, ELF_RELOC_REL, elf_obj_lookup); -#if defined(__i386__) || defined(__amd64__) - else if (ifuncs) - elf_reloc_ifunc(lf, base, rel, ELF_RELOC_REL, - elf_obj_lookup); -#endif } } @@ -1554,15 +1549,10 @@ /* Only do local relocs */ if (ELF_ST_BIND(sym->st_info) != STB_LOCAL) continue; - if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC) == - ifuncs) + if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || + elf_is_ifunc_reloc(rela->r_info)) == ifuncs) elf_reloc_local(lf, base, rela, ELF_RELOC_RELA, elf_obj_lookup); -#if defined(__i386__) || defined(__amd64__) - else if (ifuncs) - elf_reloc_ifunc(lf, base, rela, ELF_RELOC_RELA, - elf_obj_lookup); -#endif } } return (0); Index: head/sys/mips/mips/elf_machdep.c =================================================================== --- head/sys/mips/mips/elf_machdep.c +++ head/sys/mips/mips/elf_machdep.c @@ -295,6 +295,13 @@ free(r, M_TEMP); } +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, Index: head/sys/powerpc/powerpc/elf32_machdep.c =================================================================== --- head/sys/powerpc/powerpc/elf32_machdep.c +++ head/sys/powerpc/powerpc/elf32_machdep.c @@ -214,6 +214,13 @@ } #ifndef __powerpc64__ +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + /* Process one elf relocation with addend. */ static int elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data, Index: head/sys/powerpc/powerpc/elf64_machdep.c =================================================================== --- head/sys/powerpc/powerpc/elf64_machdep.c +++ head/sys/powerpc/powerpc/elf64_machdep.c @@ -276,6 +276,12 @@ *off = len; } +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} /* Process one elf relocation with addend. */ static int Index: head/sys/riscv/riscv/elf_machdep.c =================================================================== --- head/sys/riscv/riscv/elf_machdep.c +++ head/sys/riscv/riscv/elf_machdep.c @@ -259,6 +259,13 @@ return "*unknown*"; } +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + /* * Currently kernel loadable module for RISCV is compiled with -fPIC option. * (see also additional CFLAGS definition for RISCV in sys/conf/kmod.mk) Index: head/sys/sparc64/sparc64/elf_machdep.c =================================================================== --- head/sys/sparc64/sparc64/elf_machdep.c +++ head/sys/sparc64/sparc64/elf_machdep.c @@ -312,6 +312,13 @@ }; #define RELOC_VALUE_BITMASK(t) (reloc_target_bitmask[t]) +bool +elf_is_ifunc_reloc(Elf_Size r_info __unused) +{ + + return (false); +} + int elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup __unused) Index: head/sys/sys/linker.h =================================================================== --- head/sys/sys/linker.h +++ head/sys/sys/linker.h @@ -272,9 +272,8 @@ typedef int elf_lookup_fn(linker_file_t, Elf_Size, int, Elf_Addr *); /* Support functions */ +bool elf_is_ifunc_reloc(Elf_Size r_info); int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, - int _type, elf_lookup_fn _lu); -int elf_reloc_ifunc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);