Index: head/sys/compat/cloudabi/cloudabi_vdso.c =================================================================== --- head/sys/compat/cloudabi/cloudabi_vdso.c (revision 355312) +++ head/sys/compat/cloudabi/cloudabi_vdso.c (revision 355313) @@ -1,88 +1,89 @@ /*- * Copyright (c) 2016 Nuxi, https://nuxi.nl/ * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include void cloudabi_vdso_init(struct sysentvec *sv, char *begin, char *end) { vm_page_t m; vm_object_t obj; vm_offset_t addr; size_t i, pages, pages_length, vdso_length; /* Determine the number of pages needed to store the vDSO. */ vdso_length = end - begin; pages = howmany(vdso_length, PAGE_SIZE); pages_length = pages * PAGE_SIZE; /* Allocate a VM object and fill it with the vDSO. */ obj = vm_pager_allocate(OBJT_PHYS, 0, pages_length, VM_PROT_DEFAULT, 0, NULL); addr = kva_alloc(PAGE_SIZE); for (i = 0; i < pages; ++i) { VM_OBJECT_WLOCK(obj); - m = vm_page_grab(obj, i, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO); - m->valid = VM_PAGE_BITS_ALL; + m = vm_page_grab(obj, i, VM_ALLOC_ZERO); VM_OBJECT_WUNLOCK(obj); + vm_page_valid(m); + vm_page_xunbusy(m); pmap_qenter(addr, &m, 1); memcpy((void *)addr, begin + i * PAGE_SIZE, MIN(vdso_length - i * PAGE_SIZE, PAGE_SIZE)); pmap_qremove(addr, 1); } kva_free(addr, PAGE_SIZE); /* * Place the vDSO at the top of the address space. The user * stack can start right below it. */ sv->sv_shared_page_base = sv->sv_maxuser - pages_length; sv->sv_shared_page_len = pages_length; sv->sv_shared_page_obj = obj; sv->sv_usrstack = sv->sv_shared_page_base; } void cloudabi_vdso_destroy(struct sysentvec *sv) { vm_object_deallocate(sv->sv_shared_page_obj); } Index: head/sys/compat/linux/linux_vdso.c =================================================================== --- head/sys/compat/linux/linux_vdso.c (revision 355312) +++ head/sys/compat/linux/linux_vdso.c (revision 355313) @@ -1,244 +1,245 @@ /*- * Copyright (c) 2013 Dmitry Chagin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) #define __ELF_WORD_SIZE 32 #else #define __ELF_WORD_SIZE 64 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include SLIST_HEAD(, linux_vdso_sym) __elfN(linux_vdso_syms) = SLIST_HEAD_INITIALIZER(__elfN(linux_vdso_syms)); static int __elfN(symtabindex); static int __elfN(symstrindex); static void __elfN(linux_vdso_lookup)(Elf_Ehdr *, struct linux_vdso_sym *); void __elfN(linux_vdso_sym_init)(struct linux_vdso_sym *s) { SLIST_INSERT_HEAD(&__elfN(linux_vdso_syms), s, sym); } vm_object_t __elfN(linux_shared_page_init)(char **mapping) { vm_page_t m; vm_object_t obj; vm_offset_t addr; obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE, VM_PROT_DEFAULT, 0, NULL); VM_OBJECT_WLOCK(obj); - m = vm_page_grab(obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO); - m->valid = VM_PAGE_BITS_ALL; + m = vm_page_grab(obj, 0, VM_ALLOC_ZERO); VM_OBJECT_WUNLOCK(obj); + vm_page_valid(m); + vm_page_xunbusy(m); addr = kva_alloc(PAGE_SIZE); pmap_qenter(addr, &m, 1); *mapping = (char *)addr; return (obj); } void __elfN(linux_shared_page_fini)(vm_object_t obj) { vm_object_deallocate(obj); } void __elfN(linux_vdso_fixup)(struct sysentvec *sv) { Elf_Ehdr *ehdr; Elf_Shdr *shdr; int i; ehdr = (Elf_Ehdr *) sv->sv_sigcode; if (!IS_ELF(*ehdr) || ehdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || ehdr->e_ident[EI_DATA] != ELF_TARG_DATA || ehdr->e_ident[EI_VERSION] != EV_CURRENT || ehdr->e_shoff == 0 || ehdr->e_shentsize != sizeof(Elf_Shdr)) panic("Linux invalid vdso header.\n"); if (ehdr->e_type != ET_DYN) panic("Linux invalid vdso header.\n"); shdr = (Elf_Shdr *) ((caddr_t)ehdr + ehdr->e_shoff); __elfN(symtabindex) = -1; __elfN(symstrindex) = -1; for (i = 0; i < ehdr->e_shnum; i++) { if (shdr[i].sh_size == 0) continue; if (shdr[i].sh_type == SHT_DYNSYM) { __elfN(symtabindex) = i; __elfN(symstrindex) = shdr[i].sh_link; } } if (__elfN(symtabindex) == -1 || __elfN(symstrindex) == -1) panic("Linux invalid vdso header.\n"); ehdr->e_ident[EI_OSABI] = ELFOSABI_LINUX; } void __elfN(linux_vdso_reloc)(struct sysentvec *sv) { struct linux_vdso_sym *lsym; Elf_Ehdr *ehdr; Elf_Phdr *phdr; Elf_Shdr *shdr; Elf_Dyn *dyn; Elf_Sym *sym; int i, j, symcnt; ehdr = (Elf_Ehdr *) sv->sv_sigcode; /* Adjust our so relative to the sigcode_base */ if (sv->sv_shared_page_base != 0) { ehdr->e_entry += sv->sv_shared_page_base; phdr = (Elf_Phdr *)((caddr_t)ehdr + ehdr->e_phoff); /* phdrs */ for (i = 0; i < ehdr->e_phnum; i++) { phdr[i].p_vaddr += sv->sv_shared_page_base; if (phdr[i].p_type != PT_DYNAMIC) continue; dyn = (Elf_Dyn *)((caddr_t)ehdr + phdr[i].p_offset); for(; dyn->d_tag != DT_NULL; dyn++) { switch (dyn->d_tag) { case DT_PLTGOT: case DT_HASH: case DT_STRTAB: case DT_SYMTAB: case DT_RELA: case DT_INIT: case DT_FINI: case DT_REL: case DT_DEBUG: case DT_JMPREL: case DT_VERSYM: case DT_VERDEF: case DT_VERNEED: case DT_ADDRRNGLO ... DT_ADDRRNGHI: dyn->d_un.d_ptr += sv->sv_shared_page_base; break; case DT_ENCODING ... DT_LOOS-1: case DT_LOOS ... DT_HIOS: if (dyn->d_tag >= DT_ENCODING && (dyn->d_tag & 1) == 0) dyn->d_un.d_ptr += sv->sv_shared_page_base; break; default: break; } } } /* sections */ shdr = (Elf_Shdr *)((caddr_t)ehdr + ehdr->e_shoff); for(i = 0; i < ehdr->e_shnum; i++) { if (!(shdr[i].sh_flags & SHF_ALLOC)) continue; shdr[i].sh_addr += sv->sv_shared_page_base; if (shdr[i].sh_type != SHT_SYMTAB && shdr[i].sh_type != SHT_DYNSYM) continue; sym = (Elf_Sym *)((caddr_t)ehdr + shdr[i].sh_offset); symcnt = shdr[i].sh_size / sizeof(*sym); for(j = 0; j < symcnt; j++, sym++) { if (sym->st_shndx == SHN_UNDEF || sym->st_shndx == SHN_ABS) continue; sym->st_value += sv->sv_shared_page_base; } } } SLIST_FOREACH(lsym, &__elfN(linux_vdso_syms), sym) __elfN(linux_vdso_lookup)(ehdr, lsym); } static void __elfN(linux_vdso_lookup)(Elf_Ehdr *ehdr, struct linux_vdso_sym *vsym) { vm_offset_t strtab, symname; uint32_t symcnt; Elf_Shdr *shdr; int i; shdr = (Elf_Shdr *) ((caddr_t)ehdr + ehdr->e_shoff); strtab = (vm_offset_t)((caddr_t)ehdr + shdr[__elfN(symstrindex)].sh_offset); Elf_Sym *sym = (Elf_Sym *)((caddr_t)ehdr + shdr[__elfN(symtabindex)].sh_offset); symcnt = shdr[__elfN(symtabindex)].sh_size / sizeof(*sym); for (i = 0; i < symcnt; ++i, ++sym) { symname = strtab + sym->st_name; if (strncmp(vsym->symname, (char *)symname, vsym->size) == 0) { *vsym->ptr = (uintptr_t)sym->st_value; break; } } } Index: head/sys/kern/kern_sharedpage.c =================================================================== --- head/sys/kern/kern_sharedpage.c (revision 355312) +++ head/sys/kern/kern_sharedpage.c (revision 355313) @@ -1,289 +1,290 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010, 2012 Konstantin Belousov * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct sx shared_page_alloc_sx; static vm_object_t shared_page_obj; static int shared_page_free; char *shared_page_mapping; void shared_page_write(int base, int size, const void *data) { bcopy(data, shared_page_mapping + base, size); } static int shared_page_alloc_locked(int size, int align) { int res; res = roundup(shared_page_free, align); if (res + size >= IDX_TO_OFF(shared_page_obj->size)) res = -1; else shared_page_free = res + size; return (res); } int shared_page_alloc(int size, int align) { int res; sx_xlock(&shared_page_alloc_sx); res = shared_page_alloc_locked(size, align); sx_xunlock(&shared_page_alloc_sx); return (res); } int shared_page_fill(int size, int align, const void *data) { int res; sx_xlock(&shared_page_alloc_sx); res = shared_page_alloc_locked(size, align); if (res != -1) shared_page_write(res, size, data); sx_xunlock(&shared_page_alloc_sx); return (res); } static void shared_page_init(void *dummy __unused) { vm_page_t m; vm_offset_t addr; sx_init(&shared_page_alloc_sx, "shpsx"); shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE, VM_PROT_DEFAULT, 0, NULL); VM_OBJECT_WLOCK(shared_page_obj); - m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_NOBUSY | VM_ALLOC_ZERO); - m->valid = VM_PAGE_BITS_ALL; + m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_ZERO); VM_OBJECT_WUNLOCK(shared_page_obj); + vm_page_valid(m); + vm_page_xunbusy(m); addr = kva_alloc(PAGE_SIZE); pmap_qenter(addr, &m, 1); shared_page_mapping = (char *)addr; } SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init, NULL); /* * Push the timehands update to the shared page. * * The lockless update scheme is similar to the one used to update the * in-kernel timehands, see sys/kern/kern_tc.c:tc_windup() (which * calls us after the timehands are updated). */ static void timehands_update(struct vdso_sv_tk *svtk) { struct vdso_timehands th; struct vdso_timekeep *tk; uint32_t enabled, idx; enabled = tc_fill_vdso_timehands(&th); th.th_gen = 0; idx = svtk->sv_timekeep_curr; if (++idx >= VDSO_TH_NUM) idx = 0; svtk->sv_timekeep_curr = idx; if (++svtk->sv_timekeep_gen == 0) svtk->sv_timekeep_gen = 1; tk = (struct vdso_timekeep *)(shared_page_mapping + svtk->sv_timekeep_off); tk->tk_th[idx].th_gen = 0; atomic_thread_fence_rel(); if (enabled) tk->tk_th[idx] = th; atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen); atomic_store_rel_32(&tk->tk_current, idx); /* * The ordering of the assignment to tk_enabled relative to * the update of the vdso_timehands is not important. */ tk->tk_enabled = enabled; } #ifdef COMPAT_FREEBSD32 static void timehands_update32(struct vdso_sv_tk *svtk) { struct vdso_timehands32 th; struct vdso_timekeep32 *tk; uint32_t enabled, idx; enabled = tc_fill_vdso_timehands32(&th); th.th_gen = 0; idx = svtk->sv_timekeep_curr; if (++idx >= VDSO_TH_NUM) idx = 0; svtk->sv_timekeep_curr = idx; if (++svtk->sv_timekeep_gen == 0) svtk->sv_timekeep_gen = 1; tk = (struct vdso_timekeep32 *)(shared_page_mapping + svtk->sv_timekeep_off); tk->tk_th[idx].th_gen = 0; atomic_thread_fence_rel(); if (enabled) tk->tk_th[idx] = th; atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen); atomic_store_rel_32(&tk->tk_current, idx); tk->tk_enabled = enabled; } #endif /* * This is hackish, but easiest way to avoid creating list structures * that needs to be iterated over from the hardclock interrupt * context. */ static struct vdso_sv_tk *host_svtk; #ifdef COMPAT_FREEBSD32 static struct vdso_sv_tk *compat32_svtk; #endif void timekeep_push_vdso(void) { if (host_svtk != NULL) timehands_update(host_svtk); #ifdef COMPAT_FREEBSD32 if (compat32_svtk != NULL) timehands_update32(compat32_svtk); #endif } struct vdso_sv_tk * alloc_sv_tk(void) { struct vdso_sv_tk *svtk; int tk_base; uint32_t tk_ver; tk_ver = VDSO_TK_VER_CURR; svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO); tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) + sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16); KASSERT(tk_base != -1, ("tk_base -1 for native")); shared_page_write(tk_base + offsetof(struct vdso_timekeep, tk_ver), sizeof(uint32_t), &tk_ver); svtk->sv_timekeep_off = tk_base; timekeep_push_vdso(); return (svtk); } #ifdef COMPAT_FREEBSD32 struct vdso_sv_tk * alloc_sv_tk_compat32(void) { struct vdso_sv_tk *svtk; int tk_base; uint32_t tk_ver; svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO); tk_ver = VDSO_TK_VER_CURR; tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) + sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16); KASSERT(tk_base != -1, ("tk_base -1 for 32bit")); shared_page_write(tk_base + offsetof(struct vdso_timekeep32, tk_ver), sizeof(uint32_t), &tk_ver); svtk->sv_timekeep_off = tk_base; timekeep_push_vdso(); return (svtk); } #endif void exec_sysvec_init(void *param) { struct sysentvec *sv; sv = (struct sysentvec *)param; if ((sv->sv_flags & SV_SHP) == 0) return; sv->sv_shared_page_obj = shared_page_obj; sv->sv_sigcode_base = sv->sv_shared_page_base + shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode); if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD) return; if ((sv->sv_flags & SV_TIMEKEEP) != 0) { #ifdef COMPAT_FREEBSD32 if ((sv->sv_flags & SV_ILP32) != 0) { KASSERT(compat32_svtk == NULL, ("Compat32 already registered")); compat32_svtk = alloc_sv_tk_compat32(); sv->sv_timekeep_base = sv->sv_shared_page_base + compat32_svtk->sv_timekeep_off; } else { #endif KASSERT(host_svtk == NULL, ("Host already registered")); host_svtk = alloc_sv_tk(); sv->sv_timekeep_base = sv->sv_shared_page_base + host_svtk->sv_timekeep_off; #ifdef COMPAT_FREEBSD32 } #endif } }