Index: head/sys/amd64/sgx/sgx.c =================================================================== --- head/sys/amd64/sgx/sgx.c (revision 355313) +++ head/sys/amd64/sgx/sgx.c (revision 355314) @@ -1,1218 +1,1222 @@ /*- * Copyright (c) 2017 Ruslan Bukin * All rights reserved. * * This software was developed by BAE Systems, the University of Cambridge * Computer Laboratory, and Memorial University under DARPA/AFRL contract * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing * (TC) research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Design overview. * * The driver provides character device for mmap(2) and ioctl(2) system calls * allowing user to manage isolated compartments ("enclaves") in user VA space. * * The driver duties is EPC pages management, enclave management, user data * validation. * * This driver requires Intel SGX support from hardware. * * /dev/sgx: * .mmap: * sgx_mmap_single() allocates VM object with following pager * operations: * a) sgx_pg_ctor(): * VM object constructor does nothing * b) sgx_pg_dtor(): * VM object destructor destroys the SGX enclave associated * with the object: it frees all the EPC pages allocated for * enclave and removes the enclave. * c) sgx_pg_fault(): * VM object fault handler does nothing * * .ioctl: * sgx_ioctl(): * a) SGX_IOC_ENCLAVE_CREATE * Adds Enclave SECS page: initial step of enclave creation. * b) SGX_IOC_ENCLAVE_ADD_PAGE * Adds TCS, REG pages to the enclave. * c) SGX_IOC_ENCLAVE_INIT * Finalizes enclave creation. * * Enclave lifecycle: * .-- ECREATE -- Add SECS page * Kernel | EADD -- Add TCS, REG pages * space | EEXTEND -- Measure the page (take unique hash) * ENCLS | EPA -- Allocate version array page * '-- EINIT -- Finalize enclave creation * User .-- EENTER -- Go to entry point of enclave * space | EEXIT -- Exit back to main application * ENCLU '-- ERESUME -- Resume enclave execution (e.g. after exception) * * Enclave lifecycle from driver point of view: * 1) User calls mmap() on /dev/sgx: we allocate a VM object * 2) User calls ioctl SGX_IOC_ENCLAVE_CREATE: we look for the VM object * associated with user process created on step 1, create SECS physical * page and store it in enclave's VM object queue by special index * SGX_SECS_VM_OBJECT_INDEX. * 3) User calls ioctl SGX_IOC_ENCLAVE_ADD_PAGE: we look for enclave created * on step 2, create TCS or REG physical page and map it to specified by * user address of enclave VM object. * 4) User finalizes enclave creation with ioctl SGX_IOC_ENCLAVE_INIT call. * 5) User can freely enter to and exit from enclave using ENCLU instructions * from userspace: the driver does nothing here. * 6) User proceed munmap(2) system call (or the process with enclave dies): * we destroy the enclave associated with the object. * * EPC page types and their indexes in VM object queue: * - PT_SECS index is special and equals SGX_SECS_VM_OBJECT_INDEX (-1); * - PT_TCS and PT_REG indexes are specified by user in addr field of ioctl * request data and determined as follows: * pidx = OFF_TO_IDX(addp->addr - vmh->base); * - PT_VA index is special, created for PT_REG, PT_TCS and PT_SECS pages * and determined by formula: * va_page_idx = - SGX_VA_PAGES_OFFS - (page_idx / SGX_VA_PAGE_SLOTS); * PT_VA page can hold versions of up to 512 pages, and slot for each * page in PT_VA page is determined as follows: * va_slot_idx = page_idx % SGX_VA_PAGE_SLOTS; * - PT_TRIM is unused. * * Locking: * SGX ENCLS set of instructions have limitations on concurrency: * some instructions can't be executed same time on different CPUs. * We use sc->mtx_encls lock around them to prevent concurrent execution. * sc->mtx lock is used to manage list of created enclaves and the state of * SGX driver. * * Eviction of EPC pages: * Eviction support is not implemented in this driver, however the driver * manages VA (version array) pages: it allocates a VA slot for each EPC * page. This will be required for eviction support in future. * VA pages and slots are currently unused. * * Intel® 64 and IA-32 Architectures Software Developer's Manual * https://software.intel.com/en-us/articles/intel-sdm */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SGX_DEBUG #undef SGX_DEBUG #ifdef SGX_DEBUG #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) #else #define dprintf(fmt, ...) #endif static struct cdev_pager_ops sgx_pg_ops; struct sgx_softc sgx_sc; static int sgx_get_epc_page(struct sgx_softc *sc, struct epc_page **epc) { vmem_addr_t addr; int i; if (vmem_alloc(sc->vmem_epc, PAGE_SIZE, M_FIRSTFIT | M_NOWAIT, &addr) == 0) { i = (addr - sc->epc_base) / PAGE_SIZE; *epc = &sc->epc_pages[i]; return (0); } return (ENOMEM); } static void sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc) { vmem_addr_t addr; if (epc == NULL) return; addr = (epc->index * PAGE_SIZE) + sc->epc_base; vmem_free(sc->vmem_epc, addr, PAGE_SIZE); } static int sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object, uint64_t idx) { struct epc_page *epc; vm_page_t page; vm_page_t p; int ret; VM_OBJECT_ASSERT_WLOCKED(object); p = vm_page_lookup(object, idx); if (p == NULL) { ret = sgx_get_epc_page(sc, &epc); if (ret) { dprintf("%s: No free EPC pages available.\n", __func__); return (ret); } mtx_lock(&sc->mtx_encls); sgx_epa((void *)epc->base); mtx_unlock(&sc->mtx_encls); page = PHYS_TO_VM_PAGE(epc->phys); page->valid = VM_PAGE_BITS_ALL; vm_page_insert(page, object, idx); } return (0); } static int sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave, uint64_t addr) { vm_pindex_t pidx; uint64_t va_page_idx; uint64_t idx; vm_object_t object; int va_slot; int ret; object = enclave->object; VM_OBJECT_ASSERT_WLOCKED(object); pidx = OFF_TO_IDX(addr); va_slot = pidx % SGX_VA_PAGE_SLOTS; va_page_idx = pidx / SGX_VA_PAGE_SLOTS; idx = - SGX_VA_PAGES_OFFS - va_page_idx; ret = sgx_va_slot_init_by_index(sc, object, idx); return (ret); } static int sgx_mem_find(struct sgx_softc *sc, uint64_t addr, vm_map_entry_t *entry0, vm_object_t *object0) { vm_map_t map; vm_map_entry_t entry; vm_object_t object; map = &curproc->p_vmspace->vm_map; vm_map_lock_read(map); if (!vm_map_lookup_entry(map, addr, &entry)) { vm_map_unlock_read(map); dprintf("%s: Can't find enclave.\n", __func__); return (EINVAL); } object = entry->object.vm_object; if (object == NULL || object->handle == NULL) { vm_map_unlock_read(map); return (EINVAL); } if (object->type != OBJT_MGTDEVICE || object->un_pager.devp.ops != &sgx_pg_ops) { vm_map_unlock_read(map); return (EINVAL); } vm_object_reference(object); *object0 = object; *entry0 = entry; vm_map_unlock_read(map); return (0); } static int sgx_enclave_find(struct sgx_softc *sc, uint64_t addr, struct sgx_enclave **encl) { struct sgx_vm_handle *vmh; struct sgx_enclave *enclave; vm_map_entry_t entry; vm_object_t object; int ret; ret = sgx_mem_find(sc, addr, &entry, &object); if (ret) return (ret); vmh = object->handle; if (vmh == NULL) { vm_object_deallocate(object); return (EINVAL); } enclave = vmh->enclave; if (enclave == NULL || enclave->object == NULL) { vm_object_deallocate(object); return (EINVAL); } *encl = enclave; return (0); } static int sgx_enclave_alloc(struct sgx_softc *sc, struct secs *secs, struct sgx_enclave **enclave0) { struct sgx_enclave *enclave; enclave = malloc(sizeof(struct sgx_enclave), M_SGX, M_WAITOK | M_ZERO); enclave->base = secs->base; enclave->size = secs->size; *enclave0 = enclave; return (0); } static void sgx_epc_page_remove(struct sgx_softc *sc, struct epc_page *epc) { mtx_lock(&sc->mtx_encls); sgx_eremove((void *)epc->base); mtx_unlock(&sc->mtx_encls); } static void sgx_page_remove(struct sgx_softc *sc, vm_page_t p) { struct epc_page *epc; vm_paddr_t pa; uint64_t offs; (void)vm_page_remove(p); dprintf("%s: p->pidx %ld\n", __func__, p->pindex); pa = VM_PAGE_TO_PHYS(p); epc = &sc->epc_pages[0]; offs = (pa - epc->phys) / PAGE_SIZE; epc = &sc->epc_pages[offs]; sgx_epc_page_remove(sc, epc); sgx_put_epc_page(sc, epc); } static void sgx_enclave_remove(struct sgx_softc *sc, struct sgx_enclave *enclave) { vm_object_t object; vm_page_t p, p_secs, p_next; mtx_lock(&sc->mtx); TAILQ_REMOVE(&sc->enclaves, enclave, next); mtx_unlock(&sc->mtx); object = enclave->object; VM_OBJECT_WLOCK(object); /* * First remove all the pages except SECS, * then remove SECS page. */ - p_secs = NULL; +restart: TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) { - if (p->pindex == SGX_SECS_VM_OBJECT_INDEX) { - p_secs = p; + if (p->pindex == SGX_SECS_VM_OBJECT_INDEX) continue; - } + if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) + goto restart; sgx_page_remove(sc, p); } + p_secs = vm_page_grab(object, SGX_SECS_VM_OBJECT_INDEX, + VM_ALLOC_NOCREAT); /* Now remove SECS page */ if (p_secs != NULL) sgx_page_remove(sc, p_secs); KASSERT(TAILQ_EMPTY(&object->memq) == 1, ("not empty")); KASSERT(object->resident_page_count == 0, ("count")); VM_OBJECT_WUNLOCK(object); } static int sgx_measure_page(struct sgx_softc *sc, struct epc_page *secs, struct epc_page *epc, uint16_t mrmask) { int i, j; int ret; mtx_lock(&sc->mtx_encls); for (i = 0, j = 1; i < PAGE_SIZE; i += 0x100, j <<= 1) { if (!(j & mrmask)) continue; ret = sgx_eextend((void *)secs->base, (void *)(epc->base + i)); if (ret == SGX_EFAULT) { mtx_unlock(&sc->mtx_encls); return (ret); } } mtx_unlock(&sc->mtx_encls); return (0); } static int sgx_secs_validate(struct sgx_softc *sc, struct secs *secs) { struct secs_attr *attr; int i; if (secs->size == 0) return (EINVAL); /* BASEADDR must be naturally aligned on an SECS.SIZE boundary. */ if (secs->base & (secs->size - 1)) return (EINVAL); /* SECS.SIZE must be at least 2 pages. */ if (secs->size < 2 * PAGE_SIZE) return (EINVAL); if ((secs->size & (secs->size - 1)) != 0) return (EINVAL); attr = &secs->attributes; if (attr->reserved1 != 0 || attr->reserved2 != 0 || attr->reserved3 != 0) return (EINVAL); for (i = 0; i < SECS_ATTR_RSV4_SIZE; i++) if (attr->reserved4[i]) return (EINVAL); /* * Intel® Software Guard Extensions Programming Reference * 6.7.2 Relevant Fields in Various Data Structures * 6.7.2.1 SECS.ATTRIBUTES.XFRM * XFRM[1:0] must be set to 0x3. */ if ((attr->xfrm & 0x3) != 0x3) return (EINVAL); if (!attr->mode64bit) return (EINVAL); if (secs->size > sc->enclave_size_max) return (EINVAL); for (i = 0; i < SECS_RSV1_SIZE; i++) if (secs->reserved1[i]) return (EINVAL); for (i = 0; i < SECS_RSV2_SIZE; i++) if (secs->reserved2[i]) return (EINVAL); for (i = 0; i < SECS_RSV3_SIZE; i++) if (secs->reserved3[i]) return (EINVAL); for (i = 0; i < SECS_RSV4_SIZE; i++) if (secs->reserved4[i]) return (EINVAL); return (0); } static int sgx_tcs_validate(struct tcs *tcs) { int i; if ((tcs->flags) || (tcs->ossa & (PAGE_SIZE - 1)) || (tcs->ofsbasgx & (PAGE_SIZE - 1)) || (tcs->ogsbasgx & (PAGE_SIZE - 1)) || ((tcs->fslimit & 0xfff) != 0xfff) || ((tcs->gslimit & 0xfff) != 0xfff)) return (EINVAL); for (i = 0; i < nitems(tcs->reserved3); i++) if (tcs->reserved3[i]) return (EINVAL); return (0); } static void sgx_tcs_dump(struct sgx_softc *sc, struct tcs *t) { dprintf("t->flags %lx\n", t->flags); dprintf("t->ossa %lx\n", t->ossa); dprintf("t->cssa %x\n", t->cssa); dprintf("t->nssa %x\n", t->nssa); dprintf("t->oentry %lx\n", t->oentry); dprintf("t->ofsbasgx %lx\n", t->ofsbasgx); dprintf("t->ogsbasgx %lx\n", t->ogsbasgx); dprintf("t->fslimit %x\n", t->fslimit); dprintf("t->gslimit %x\n", t->gslimit); } static int sgx_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { struct sgx_vm_handle *vmh; vmh = handle; if (vmh == NULL) { dprintf("%s: vmh not found.\n", __func__); return (0); } dprintf("%s: vmh->base %lx foff 0x%lx size 0x%lx\n", __func__, vmh->base, foff, size); return (0); } static void sgx_pg_dtor(void *handle) { struct sgx_vm_handle *vmh; struct sgx_softc *sc; vmh = handle; if (vmh == NULL) { dprintf("%s: vmh not found.\n", __func__); return; } sc = vmh->sc; if (sc == NULL) { dprintf("%s: sc is NULL\n", __func__); return; } if (vmh->enclave == NULL) { dprintf("%s: Enclave not found.\n", __func__); return; } sgx_enclave_remove(sc, vmh->enclave); free(vmh->enclave, M_SGX); free(vmh, M_SGX); } static int sgx_pg_fault(vm_object_t object, vm_ooffset_t offset, int prot, vm_page_t *mres) { /* * The purpose of this trivial handler is to handle the race * when user tries to access mmaped region before or during * enclave creation ioctl calls. */ dprintf("%s: offset 0x%lx\n", __func__, offset); return (VM_PAGER_FAIL); } static struct cdev_pager_ops sgx_pg_ops = { .cdev_pg_ctor = sgx_pg_ctor, .cdev_pg_dtor = sgx_pg_dtor, .cdev_pg_fault = sgx_pg_fault, }; static void sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object, vm_pindex_t pidx) { VM_OBJECT_ASSERT_WLOCKED(object); page->valid = VM_PAGE_BITS_ALL; vm_page_insert(page, object, pidx); } static void sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc, uint64_t addr) { vm_pindex_t pidx; vm_page_t page; VM_OBJECT_ASSERT_WLOCKED(enclave->object); pidx = OFF_TO_IDX(addr); page = PHYS_TO_VM_PAGE(epc->phys); sgx_insert_epc_page_by_index(page, enclave->object, pidx); } static int sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param) { struct sgx_vm_handle *vmh; vm_map_entry_t entry; vm_page_t p; struct page_info pginfo; struct secinfo secinfo; struct sgx_enclave *enclave; struct epc_page *epc; struct secs *secs; vm_object_t object; vm_page_t page; int ret; epc = NULL; secs = NULL; enclave = NULL; object = NULL; /* SGX Enclave Control Structure (SECS) */ secs = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); ret = copyin((void *)param->src, secs, sizeof(struct secs)); if (ret) { dprintf("%s: Can't copy SECS.\n", __func__); goto error; } ret = sgx_secs_validate(sc, secs); if (ret) { dprintf("%s: SECS validation failed.\n", __func__); goto error; } ret = sgx_mem_find(sc, secs->base, &entry, &object); if (ret) { dprintf("%s: Can't find vm_map.\n", __func__); goto error; } vmh = object->handle; if (!vmh) { dprintf("%s: Can't find vmh.\n", __func__); ret = ENXIO; goto error; } dprintf("%s: entry start %lx offset %lx\n", __func__, entry->start, entry->offset); vmh->base = (entry->start - entry->offset); ret = sgx_enclave_alloc(sc, secs, &enclave); if (ret) { dprintf("%s: Can't alloc enclave.\n", __func__); goto error; } enclave->object = object; enclave->vmh = vmh; memset(&secinfo, 0, sizeof(struct secinfo)); memset(&pginfo, 0, sizeof(struct page_info)); pginfo.linaddr = 0; pginfo.srcpge = (uint64_t)secs; pginfo.secinfo = &secinfo; pginfo.secs = 0; ret = sgx_get_epc_page(sc, &epc); if (ret) { dprintf("%s: Failed to get free epc page.\n", __func__); goto error; } enclave->secs_epc_page = epc; VM_OBJECT_WLOCK(object); p = vm_page_lookup(object, SGX_SECS_VM_OBJECT_INDEX); if (p) { VM_OBJECT_WUNLOCK(object); /* SECS page already added. */ ret = ENXIO; goto error; } ret = sgx_va_slot_init_by_index(sc, object, - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX); if (ret) { VM_OBJECT_WUNLOCK(object); dprintf("%s: Can't init va slot.\n", __func__); goto error; } mtx_lock(&sc->mtx); if ((sc->state & SGX_STATE_RUNNING) == 0) { mtx_unlock(&sc->mtx); /* Remove VA page that was just created for SECS page. */ - p = vm_page_lookup(enclave->object, - - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX); + p = vm_page_grab(enclave->object, + - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, + VM_ALLOC_NOCREAT); sgx_page_remove(sc, p); VM_OBJECT_WUNLOCK(object); goto error; } mtx_lock(&sc->mtx_encls); ret = sgx_ecreate(&pginfo, (void *)epc->base); mtx_unlock(&sc->mtx_encls); if (ret == SGX_EFAULT) { dprintf("%s: gp fault\n", __func__); mtx_unlock(&sc->mtx); /* Remove VA page that was just created for SECS page. */ - p = vm_page_lookup(enclave->object, - - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX); + p = vm_page_grab(enclave->object, + - SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, + VM_ALLOC_NOCREAT); sgx_page_remove(sc, p); VM_OBJECT_WUNLOCK(object); goto error; } TAILQ_INSERT_TAIL(&sc->enclaves, enclave, next); mtx_unlock(&sc->mtx); vmh->enclave = enclave; page = PHYS_TO_VM_PAGE(epc->phys); sgx_insert_epc_page_by_index(page, enclave->object, SGX_SECS_VM_OBJECT_INDEX); VM_OBJECT_WUNLOCK(object); /* Release the reference. */ vm_object_deallocate(object); free(secs, M_SGX); return (0); error: free(secs, M_SGX); sgx_put_epc_page(sc, epc); free(enclave, M_SGX); vm_object_deallocate(object); return (ret); } static int sgx_ioctl_add_page(struct sgx_softc *sc, struct sgx_enclave_add_page *addp) { struct epc_page *secs_epc_page; struct sgx_enclave *enclave; struct sgx_vm_handle *vmh; struct epc_page *epc; struct page_info pginfo; struct secinfo secinfo; vm_object_t object; void *tmp_vaddr; uint64_t page_type; struct tcs *t; uint64_t addr; uint64_t pidx; vm_page_t p; int ret; tmp_vaddr = NULL; epc = NULL; object = NULL; /* Find and get reference to VM object. */ ret = sgx_enclave_find(sc, addp->addr, &enclave); if (ret) { dprintf("%s: Failed to find enclave.\n", __func__); goto error; } object = enclave->object; KASSERT(object != NULL, ("vm object is NULL\n")); vmh = object->handle; ret = sgx_get_epc_page(sc, &epc); if (ret) { dprintf("%s: Failed to get free epc page.\n", __func__); goto error; } memset(&secinfo, 0, sizeof(struct secinfo)); ret = copyin((void *)addp->secinfo, &secinfo, sizeof(struct secinfo)); if (ret) { dprintf("%s: Failed to copy secinfo.\n", __func__); goto error; } tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); ret = copyin((void *)addp->src, tmp_vaddr, PAGE_SIZE); if (ret) { dprintf("%s: Failed to copy page.\n", __func__); goto error; } page_type = (secinfo.flags & SECINFO_FLAGS_PT_M) >> SECINFO_FLAGS_PT_S; if (page_type != SGX_PT_TCS && page_type != SGX_PT_REG) { dprintf("%s: page can't be added.\n", __func__); goto error; } if (page_type == SGX_PT_TCS) { t = (struct tcs *)tmp_vaddr; ret = sgx_tcs_validate(t); if (ret) { dprintf("%s: TCS page validation failed.\n", __func__); goto error; } sgx_tcs_dump(sc, t); } addr = (addp->addr - vmh->base); pidx = OFF_TO_IDX(addr); VM_OBJECT_WLOCK(object); p = vm_page_lookup(object, pidx); if (p) { VM_OBJECT_WUNLOCK(object); /* Page already added. */ ret = ENXIO; goto error; } ret = sgx_va_slot_init(sc, enclave, addr); if (ret) { VM_OBJECT_WUNLOCK(object); dprintf("%s: Can't init va slot.\n", __func__); goto error; } secs_epc_page = enclave->secs_epc_page; memset(&pginfo, 0, sizeof(struct page_info)); pginfo.linaddr = (uint64_t)addp->addr; pginfo.srcpge = (uint64_t)tmp_vaddr; pginfo.secinfo = &secinfo; pginfo.secs = (uint64_t)secs_epc_page->base; mtx_lock(&sc->mtx_encls); ret = sgx_eadd(&pginfo, (void *)epc->base); if (ret == SGX_EFAULT) { dprintf("%s: gp fault on eadd\n", __func__); mtx_unlock(&sc->mtx_encls); VM_OBJECT_WUNLOCK(object); goto error; } mtx_unlock(&sc->mtx_encls); ret = sgx_measure_page(sc, enclave->secs_epc_page, epc, addp->mrmask); if (ret == SGX_EFAULT) { dprintf("%s: gp fault on eextend\n", __func__); sgx_epc_page_remove(sc, epc); VM_OBJECT_WUNLOCK(object); goto error; } sgx_insert_epc_page(enclave, epc, addr); VM_OBJECT_WUNLOCK(object); /* Release the reference. */ vm_object_deallocate(object); free(tmp_vaddr, M_SGX); return (0); error: free(tmp_vaddr, M_SGX); sgx_put_epc_page(sc, epc); vm_object_deallocate(object); return (ret); } static int sgx_ioctl_init(struct sgx_softc *sc, struct sgx_enclave_init *initp) { struct epc_page *secs_epc_page; struct sgx_enclave *enclave; struct thread *td; void *tmp_vaddr; void *einittoken; void *sigstruct; vm_object_t object; int retry; int ret; td = curthread; tmp_vaddr = NULL; object = NULL; dprintf("%s: addr %lx, sigstruct %lx, einittoken %lx\n", __func__, initp->addr, initp->sigstruct, initp->einittoken); /* Find and get reference to VM object. */ ret = sgx_enclave_find(sc, initp->addr, &enclave); if (ret) { dprintf("%s: Failed to find enclave.\n", __func__); goto error; } object = enclave->object; tmp_vaddr = malloc(PAGE_SIZE, M_SGX, M_WAITOK | M_ZERO); sigstruct = tmp_vaddr; einittoken = (void *)((uint64_t)sigstruct + PAGE_SIZE / 2); ret = copyin((void *)initp->sigstruct, sigstruct, SGX_SIGSTRUCT_SIZE); if (ret) { dprintf("%s: Failed to copy SIGSTRUCT page.\n", __func__); goto error; } ret = copyin((void *)initp->einittoken, einittoken, SGX_EINITTOKEN_SIZE); if (ret) { dprintf("%s: Failed to copy EINITTOKEN page.\n", __func__); goto error; } secs_epc_page = enclave->secs_epc_page; retry = 16; do { mtx_lock(&sc->mtx_encls); ret = sgx_einit(sigstruct, (void *)secs_epc_page->base, einittoken); mtx_unlock(&sc->mtx_encls); dprintf("%s: sgx_einit returned %d\n", __func__, ret); } while (ret == SGX_UNMASKED_EVENT && retry--); if (ret) { dprintf("%s: Failed init enclave: %d\n", __func__, ret); td->td_retval[0] = ret; ret = 0; } error: free(tmp_vaddr, M_SGX); /* Release the reference. */ vm_object_deallocate(object); return (ret); } static int sgx_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { struct sgx_enclave_add_page *addp; struct sgx_enclave_create *param; struct sgx_enclave_init *initp; struct sgx_softc *sc; int ret; int len; sc = &sgx_sc; len = IOCPARM_LEN(cmd); dprintf("%s: cmd %lx, addr %lx, len %d\n", __func__, cmd, (uint64_t)addr, len); if (len > SGX_IOCTL_MAX_DATA_LEN) return (EINVAL); switch (cmd) { case SGX_IOC_ENCLAVE_CREATE: param = (struct sgx_enclave_create *)addr; ret = sgx_ioctl_create(sc, param); break; case SGX_IOC_ENCLAVE_ADD_PAGE: addp = (struct sgx_enclave_add_page *)addr; ret = sgx_ioctl_add_page(sc, addp); break; case SGX_IOC_ENCLAVE_INIT: initp = (struct sgx_enclave_init *)addr; ret = sgx_ioctl_init(sc, initp); break; default: return (EINVAL); } return (ret); } static int sgx_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize, struct vm_object **objp, int nprot) { struct sgx_vm_handle *vmh; struct sgx_softc *sc; sc = &sgx_sc; dprintf("%s: mapsize 0x%lx, offset %lx\n", __func__, mapsize, *offset); vmh = malloc(sizeof(struct sgx_vm_handle), M_SGX, M_WAITOK | M_ZERO); vmh->sc = sc; vmh->size = mapsize; vmh->mem = cdev_pager_allocate(vmh, OBJT_MGTDEVICE, &sgx_pg_ops, mapsize, nprot, *offset, NULL); if (vmh->mem == NULL) { free(vmh, M_SGX); return (ENOMEM); } VM_OBJECT_WLOCK(vmh->mem); vm_object_set_flag(vmh->mem, OBJ_PG_DTOR); VM_OBJECT_WUNLOCK(vmh->mem); *objp = vmh->mem; return (0); } static struct cdevsw sgx_cdevsw = { .d_version = D_VERSION, .d_ioctl = sgx_ioctl, .d_mmap_single = sgx_mmap_single, .d_name = "Intel SGX", }; static int sgx_get_epc_area(struct sgx_softc *sc) { vm_offset_t epc_base_vaddr; u_int cp[4]; int error; int i; cpuid_count(SGX_CPUID, 0x2, cp); sc->epc_base = ((uint64_t)(cp[1] & 0xfffff) << 32) + (cp[0] & 0xfffff000); sc->epc_size = ((uint64_t)(cp[3] & 0xfffff) << 32) + (cp[2] & 0xfffff000); sc->npages = sc->epc_size / SGX_PAGE_SIZE; if (sc->epc_size == 0 || sc->epc_base == 0) { printf("%s: Incorrect EPC data: EPC base %lx, size %lu\n", __func__, sc->epc_base, sc->epc_size); return (EINVAL); } if (cp[3] & 0xffff) sc->enclave_size_max = (1 << ((cp[3] >> 8) & 0xff)); else sc->enclave_size_max = SGX_ENCL_SIZE_MAX_DEF; epc_base_vaddr = (vm_offset_t)pmap_mapdev_attr(sc->epc_base, sc->epc_size, VM_MEMATTR_DEFAULT); sc->epc_pages = malloc(sizeof(struct epc_page) * sc->npages, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < sc->npages; i++) { sc->epc_pages[i].base = epc_base_vaddr + SGX_PAGE_SIZE * i; sc->epc_pages[i].phys = sc->epc_base + SGX_PAGE_SIZE * i; sc->epc_pages[i].index = i; } sc->vmem_epc = vmem_create("SGX EPC", sc->epc_base, sc->epc_size, PAGE_SIZE, PAGE_SIZE, M_FIRSTFIT | M_WAITOK); if (sc->vmem_epc == NULL) { printf("%s: Can't create vmem arena.\n", __func__); free(sc->epc_pages, M_SGX); return (EINVAL); } error = vm_phys_fictitious_reg_range(sc->epc_base, sc->epc_base + sc->epc_size, VM_MEMATTR_DEFAULT); if (error) { printf("%s: Can't register fictitious space.\n", __func__); free(sc->epc_pages, M_SGX); return (EINVAL); } return (0); } static void sgx_put_epc_area(struct sgx_softc *sc) { vm_phys_fictitious_unreg_range(sc->epc_base, sc->epc_base + sc->epc_size); free(sc->epc_pages, M_SGX); } static int sgx_load(void) { struct sgx_softc *sc; int error; sc = &sgx_sc; if ((cpu_stdext_feature & CPUID_STDEXT_SGX) == 0) return (ENXIO); error = sgx_get_epc_area(sc); if (error) { printf("%s: Failed to get Processor Reserved Memory area.\n", __func__); return (ENXIO); } mtx_init(&sc->mtx_encls, "SGX ENCLS", NULL, MTX_DEF); mtx_init(&sc->mtx, "SGX driver", NULL, MTX_DEF); TAILQ_INIT(&sc->enclaves); sc->sgx_cdev = make_dev(&sgx_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "isgx"); sc->state |= SGX_STATE_RUNNING; printf("SGX initialized: EPC base 0x%lx size %ld (%d pages)\n", sc->epc_base, sc->epc_size, sc->npages); return (0); } static int sgx_unload(void) { struct sgx_softc *sc; sc = &sgx_sc; if ((sc->state & SGX_STATE_RUNNING) == 0) return (0); mtx_lock(&sc->mtx); if (!TAILQ_EMPTY(&sc->enclaves)) { mtx_unlock(&sc->mtx); return (EBUSY); } sc->state &= ~SGX_STATE_RUNNING; mtx_unlock(&sc->mtx); destroy_dev(sc->sgx_cdev); vmem_destroy(sc->vmem_epc); sgx_put_epc_area(sc); mtx_destroy(&sc->mtx_encls); mtx_destroy(&sc->mtx); return (0); } static int sgx_handler(module_t mod, int what, void *arg) { int error; switch (what) { case MOD_LOAD: error = sgx_load(); break; case MOD_UNLOAD: error = sgx_unload(); break; default: error = 0; break; } return (error); } static moduledata_t sgx_kmod = { "sgx", sgx_handler, NULL }; DECLARE_MODULE(sgx, sgx_kmod, SI_SUB_LAST, SI_ORDER_ANY); MODULE_VERSION(sgx, 1); Index: head/sys/arm/nvidia/drm2/tegra_bo.c =================================================================== --- head/sys/arm/nvidia/drm2/tegra_bo.c (revision 355313) +++ head/sys/arm/nvidia/drm2/tegra_bo.c (revision 355314) @@ -1,367 +1,368 @@ /*- * Copyright (c) 2015 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void tegra_bo_destruct(struct tegra_bo *bo) { vm_page_t m; size_t size; int i; if (bo->cdev_pager == NULL) return; size = round_page(bo->gem_obj.size); if (bo->vbase != 0) pmap_qremove(bo->vbase, bo->npages); VM_OBJECT_WLOCK(bo->cdev_pager); for (i = 0; i < bo->npages; i++) { m = bo->m[i]; + vm_page_busy_acquire(m, 0); cdev_pager_free_page(bo->cdev_pager, m); m->flags &= ~PG_FICTITIOUS; vm_page_unwire_noq(m); vm_page_free(m); } VM_OBJECT_WUNLOCK(bo->cdev_pager); vm_object_deallocate(bo->cdev_pager); if (bo->vbase != 0) vmem_free(kmem_arena, bo->vbase, size); } static void tegra_bo_free_object(struct drm_gem_object *gem_obj) { struct tegra_bo *bo; bo = container_of(gem_obj, struct tegra_bo, gem_obj); drm_gem_free_mmap_offset(gem_obj); drm_gem_object_release(gem_obj); tegra_bo_destruct(bo); free(bo->m, DRM_MEM_DRIVER); free(bo, DRM_MEM_DRIVER); } static int tegra_bo_alloc_contig(size_t npages, u_long alignment, vm_memattr_t memattr, vm_page_t **ret_page) { vm_page_t m; int pflags, tries, i; vm_paddr_t low, high, boundary; low = 0; high = -1UL; boundary = 0; pflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED | VM_ALLOC_ZERO; tries = 0; retry: m = vm_page_alloc_contig(NULL, 0, pflags, npages, low, high, alignment, boundary, memattr); if (m == NULL) { if (tries < 3) { if (!vm_page_reclaim_contig(pflags, npages, low, high, alignment, boundary)) vm_wait(NULL); tries++; goto retry; } return (ENOMEM); } for (i = 0; i < npages; i++, m++) { if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL; (*ret_page)[i] = m; } return (0); } /* Initialize pager and insert all object pages to it*/ static int tegra_bo_init_pager(struct tegra_bo *bo) { vm_page_t m; size_t size; int i; size = round_page(bo->gem_obj.size); bo->pbase = VM_PAGE_TO_PHYS(bo->m[0]); if (vmem_alloc(kmem_arena, size, M_WAITOK | M_BESTFIT, &bo->vbase)) return (ENOMEM); VM_OBJECT_WLOCK(bo->cdev_pager); for (i = 0; i < bo->npages; i++) { m = bo->m[i]; /* * XXX This is a temporary hack. * We need pager suitable for paging (mmap) managed * real (non-fictitious) pages. * - managed pages are needed for clean module unload. * - aliasing fictitious page to real one is bad, * pmap cannot handle this situation without issues * It expects that * paddr = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(paddr)) * for every single page passed to pmap. */ m->oflags &= ~VPO_UNMANAGED; m->flags |= PG_FICTITIOUS; if (vm_page_insert(m, bo->cdev_pager, i) != 0) return (EINVAL); } VM_OBJECT_WUNLOCK(bo->cdev_pager); pmap_qenter(bo->vbase, bo->m, bo->npages); return (0); } /* Allocate memory for frame buffer */ static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo) { size_t size; int rv; size = bo->gem_obj.size; bo->npages = atop(size); bo->m = malloc(sizeof(vm_page_t *) * bo->npages, DRM_MEM_DRIVER, M_WAITOK | M_ZERO); rv = tegra_bo_alloc_contig(bo->npages, PAGE_SIZE, VM_MEMATTR_WRITE_COMBINING, &(bo->m)); if (rv != 0) { DRM_WARNING("Cannot allocate memory for gem object.\n"); return (rv); } rv = tegra_bo_init_pager(bo); if (rv != 0) { DRM_WARNING("Cannot initialize gem object pager.\n"); return (rv); } return (0); } int tegra_bo_create(struct drm_device *drm, size_t size, struct tegra_bo **res_bo) { struct tegra_bo *bo; int rv; if (size <= 0) return (-EINVAL); bo = malloc(sizeof(*bo), DRM_MEM_DRIVER, M_WAITOK | M_ZERO); size = round_page(size); rv = drm_gem_object_init(drm, &bo->gem_obj, size); if (rv != 0) { free(bo, DRM_MEM_DRIVER); return (rv); } rv = drm_gem_create_mmap_offset(&bo->gem_obj); if (rv != 0) { drm_gem_object_release(&bo->gem_obj); free(bo, DRM_MEM_DRIVER); return (rv); } bo->cdev_pager = cdev_pager_allocate(&bo->gem_obj, OBJT_MGTDEVICE, drm->driver->gem_pager_ops, size, 0, 0, NULL); rv = tegra_bo_alloc(drm, bo); if (rv != 0) { tegra_bo_free_object(&bo->gem_obj); return (rv); } *res_bo = bo; return (0); } static int tegra_bo_create_with_handle(struct drm_file *file, struct drm_device *drm, size_t size, uint32_t *handle, struct tegra_bo **res_bo) { int rv; struct tegra_bo *bo; rv = tegra_bo_create(drm, size, &bo); if (rv != 0) return (rv); rv = drm_gem_handle_create(file, &bo->gem_obj, handle); if (rv != 0) { tegra_bo_free_object(&bo->gem_obj); drm_gem_object_release(&bo->gem_obj); return (rv); } drm_gem_object_unreference_unlocked(&bo->gem_obj); *res_bo = bo; return (0); } static int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm_dev, struct drm_mode_create_dumb *args) { struct tegra_drm *drm; struct tegra_bo *bo; int rv; drm = container_of(drm_dev, struct tegra_drm, drm_dev); args->pitch= (args->width * args->bpp + 7) / 8; args->pitch = roundup(args->pitch, drm->pitch_align); args->size = args->pitch * args->height; rv = tegra_bo_create_with_handle(file, drm_dev, args->size, &args->handle, &bo); return (rv); } static int tegra_bo_dumb_map_offset(struct drm_file *file_priv, struct drm_device *drm_dev, uint32_t handle, uint64_t *offset) { struct drm_gem_object *gem_obj; int rv; DRM_LOCK(drm_dev); gem_obj = drm_gem_object_lookup(drm_dev, file_priv, handle); if (gem_obj == NULL) { device_printf(drm_dev->dev, "Object not found\n"); DRM_UNLOCK(drm_dev); return (-EINVAL); } rv = drm_gem_create_mmap_offset(gem_obj); if (rv != 0) goto fail; *offset = DRM_GEM_MAPPING_OFF(gem_obj->map_list.key) | DRM_GEM_MAPPING_KEY; drm_gem_object_unreference(gem_obj); DRM_UNLOCK(drm_dev); return (0); fail: drm_gem_object_unreference(gem_obj); DRM_UNLOCK(drm_dev); return (rv); } static int tegra_bo_dumb_destroy(struct drm_file *file_priv, struct drm_device *drm_dev, unsigned int handle) { int rv; rv = drm_gem_handle_delete(file_priv, handle); return (rv); } /* * mmap support */ static int tegra_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) { #ifdef DRM_PAGER_DEBUG DRM_DEBUG("object %p offset %jd prot %d mres %p\n", vm_obj, (intmax_t)offset, prot, mres); #endif return (VM_PAGER_FAIL); } static int tegra_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { if (color != NULL) *color = 0; return (0); } static void tegra_gem_pager_dtor(void *handle) { } static struct cdev_pager_ops tegra_gem_pager_ops = { .cdev_pg_fault = tegra_gem_pager_fault, .cdev_pg_ctor = tegra_gem_pager_ctor, .cdev_pg_dtor = tegra_gem_pager_dtor }; /* Fill up relevant fields in drm_driver ops */ void tegra_bo_driver_register(struct drm_driver *drm_drv) { drm_drv->gem_free_object = tegra_bo_free_object; drm_drv->gem_pager_ops = &tegra_gem_pager_ops; drm_drv->dumb_create = tegra_bo_dumb_create; drm_drv->dumb_map_offset = tegra_bo_dumb_map_offset; drm_drv->dumb_destroy = tegra_bo_dumb_destroy; } Index: head/sys/dev/md/md.c =================================================================== --- head/sys/dev/md/md.c (revision 355313) +++ head/sys/dev/md/md.c (revision 355314) @@ -1,2182 +1,2174 @@ /*- * SPDX-License-Identifier: (Beerware AND BSD-3-Clause) * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * * $FreeBSD$ * */ /*- * The following functions are based in the vn(4) driver: mdstart_swap(), * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(), * and as such under the following copyright: * * Copyright (c) 1988 University of Utah. * Copyright (c) 1990, 1993 * The Regents of the University of California. All rights reserved. * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: Utah Hdr: vn.c 1.13 94/04/02 * * from: @(#)vn.c 8.6 (Berkeley) 4/1/94 * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03 */ #include "opt_rootdevname.h" #include "opt_geom.h" #include "opt_md.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MD_MODVER 1 #define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */ #define MD_EXITING 0x20000 /* Worker thread is exiting. */ #define MD_PROVIDERGONE 0x40000 /* Safe to free the softc */ #ifndef MD_NSECT #define MD_NSECT (10000 * 2) #endif struct md_req { unsigned md_unit; /* unit number */ enum md_types md_type; /* type of disk */ off_t md_mediasize; /* size of disk in bytes */ unsigned md_sectorsize; /* sectorsize */ unsigned md_options; /* options */ int md_fwheads; /* firmware heads */ int md_fwsectors; /* firmware sectors */ char *md_file; /* pathname of file to mount */ enum uio_seg md_file_seg; /* location of md_file */ char *md_label; /* label of the device (userspace) */ int *md_units; /* pointer to units array (kernel) */ size_t md_units_nitems; /* items in md_units array */ }; #ifdef COMPAT_FREEBSD32 struct md_ioctl32 { unsigned md_version; unsigned md_unit; enum md_types md_type; uint32_t md_file; off_t md_mediasize; unsigned md_sectorsize; unsigned md_options; uint64_t md_base; int md_fwheads; int md_fwsectors; uint32_t md_label; int md_pad[MDNPAD]; } __attribute__((__packed__)); CTASSERT((sizeof(struct md_ioctl32)) == 436); #define MDIOCATTACH_32 _IOC_NEWTYPE(MDIOCATTACH, struct md_ioctl32) #define MDIOCDETACH_32 _IOC_NEWTYPE(MDIOCDETACH, struct md_ioctl32) #define MDIOCQUERY_32 _IOC_NEWTYPE(MDIOCQUERY, struct md_ioctl32) #define MDIOCRESIZE_32 _IOC_NEWTYPE(MDIOCRESIZE, struct md_ioctl32) #endif /* COMPAT_FREEBSD32 */ static MALLOC_DEFINE(M_MD, "md_disk", "Memory Disk"); static MALLOC_DEFINE(M_MDSECT, "md_sectors", "Memory Disk Sectors"); static int md_debug; SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0, "Enable md(4) debug messages"); static int md_malloc_wait; SYSCTL_INT(_vm, OID_AUTO, md_malloc_wait, CTLFLAG_RW, &md_malloc_wait, 0, "Allow malloc to wait for memory allocations"); #if defined(MD_ROOT) && !defined(MD_ROOT_FSTYPE) #define MD_ROOT_FSTYPE "ufs" #endif #if defined(MD_ROOT) /* * Preloaded image gets put here. */ #if defined(MD_ROOT_SIZE) /* * We put the mfs_root symbol into the oldmfs section of the kernel object file. * Applications that patch the object with the image can determine * the size looking at the oldmfs section size within the kernel. */ u_char mfs_root[MD_ROOT_SIZE*1024] __attribute__ ((section ("oldmfs"))); const int mfs_root_size = sizeof(mfs_root); #elif defined(MD_ROOT_MEM) /* MD region already mapped in the memory */ u_char *mfs_root; int mfs_root_size; #else extern volatile u_char __weak_symbol mfs_root; extern volatile u_char __weak_symbol mfs_root_end; __GLOBL(mfs_root); __GLOBL(mfs_root_end); #define mfs_root_size ((uintptr_t)(&mfs_root_end - &mfs_root)) #endif #endif static g_init_t g_md_init; static g_fini_t g_md_fini; static g_start_t g_md_start; static g_access_t g_md_access; static void g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp); static g_provgone_t g_md_providergone; static struct cdev *status_dev = NULL; static struct sx md_sx; static struct unrhdr *md_uh; static d_ioctl_t mdctlioctl; static struct cdevsw mdctl_cdevsw = { .d_version = D_VERSION, .d_ioctl = mdctlioctl, .d_name = MD_NAME, }; struct g_class g_md_class = { .name = "MD", .version = G_VERSION, .init = g_md_init, .fini = g_md_fini, .start = g_md_start, .access = g_md_access, .dumpconf = g_md_dumpconf, .providergone = g_md_providergone, }; DECLARE_GEOM_CLASS(g_md_class, g_md); static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(md_softc_list); #define NINDIR (PAGE_SIZE / sizeof(uintptr_t)) #define NMASK (NINDIR-1) static int nshift; static uma_zone_t md_pbuf_zone; struct indir { uintptr_t *array; u_int total; u_int used; u_int shift; }; struct md_s { int unit; LIST_ENTRY(md_s) list; struct bio_queue_head bio_queue; struct mtx queue_mtx; struct mtx stat_mtx; struct cdev *dev; enum md_types type; off_t mediasize; unsigned sectorsize; unsigned opencount; unsigned fwheads; unsigned fwsectors; char ident[32]; unsigned flags; char name[20]; struct proc *procp; struct g_geom *gp; struct g_provider *pp; int (*start)(struct md_s *sc, struct bio *bp); struct devstat *devstat; /* MD_MALLOC related fields */ struct indir *indir; uma_zone_t uma; /* MD_PRELOAD related fields */ u_char *pl_ptr; size_t pl_len; /* MD_VNODE related fields */ struct vnode *vnode; char file[PATH_MAX]; char label[PATH_MAX]; struct ucred *cred; /* MD_SWAP related fields */ vm_object_t object; }; static struct indir * new_indir(u_int shift) { struct indir *ip; ip = malloc(sizeof *ip, M_MD, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (ip == NULL) return (NULL); ip->array = malloc(sizeof(uintptr_t) * NINDIR, M_MDSECT, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (ip->array == NULL) { free(ip, M_MD); return (NULL); } ip->total = NINDIR; ip->shift = shift; return (ip); } static void del_indir(struct indir *ip) { free(ip->array, M_MDSECT); free(ip, M_MD); } static void destroy_indir(struct md_s *sc, struct indir *ip) { int i; for (i = 0; i < NINDIR; i++) { if (!ip->array[i]) continue; if (ip->shift) destroy_indir(sc, (struct indir*)(ip->array[i])); else if (ip->array[i] > 255) uma_zfree(sc->uma, (void *)(ip->array[i])); } del_indir(ip); } /* * This function does the math and allocates the top level "indir" structure * for a device of "size" sectors. */ static struct indir * dimension(off_t size) { off_t rcnt; struct indir *ip; int layer; rcnt = size; layer = 0; while (rcnt > NINDIR) { rcnt /= NINDIR; layer++; } /* * XXX: the top layer is probably not fully populated, so we allocate * too much space for ip->array in here. */ ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO); ip->array = malloc(sizeof(uintptr_t) * NINDIR, M_MDSECT, M_WAITOK | M_ZERO); ip->total = NINDIR; ip->shift = layer * nshift; return (ip); } /* * Read a given sector */ static uintptr_t s_read(struct indir *ip, off_t offset) { struct indir *cip; int idx; uintptr_t up; if (md_debug > 1) printf("s_read(%jd)\n", (intmax_t)offset); up = 0; for (cip = ip; cip != NULL;) { if (cip->shift) { idx = (offset >> cip->shift) & NMASK; up = cip->array[idx]; cip = (struct indir *)up; continue; } idx = offset & NMASK; return (cip->array[idx]); } return (0); } /* * Write a given sector, prune the tree if the value is 0 */ static int s_write(struct indir *ip, off_t offset, uintptr_t ptr) { struct indir *cip, *lip[10]; int idx, li; uintptr_t up; if (md_debug > 1) printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr); up = 0; li = 0; cip = ip; for (;;) { lip[li++] = cip; if (cip->shift) { idx = (offset >> cip->shift) & NMASK; up = cip->array[idx]; if (up != 0) { cip = (struct indir *)up; continue; } /* Allocate branch */ cip->array[idx] = (uintptr_t)new_indir(cip->shift - nshift); if (cip->array[idx] == 0) return (ENOSPC); cip->used++; up = cip->array[idx]; cip = (struct indir *)up; continue; } /* leafnode */ idx = offset & NMASK; up = cip->array[idx]; if (up != 0) cip->used--; cip->array[idx] = ptr; if (ptr != 0) cip->used++; break; } if (cip->used != 0 || li == 1) return (0); li--; while (cip->used == 0 && cip != ip) { li--; idx = (offset >> lip[li]->shift) & NMASK; up = lip[li]->array[idx]; KASSERT(up == (uintptr_t)cip, ("md screwed up")); del_indir(cip); lip[li]->array[idx] = 0; lip[li]->used--; cip = lip[li]; } return (0); } static int g_md_access(struct g_provider *pp, int r, int w, int e) { struct md_s *sc; sc = pp->geom->softc; if (sc == NULL) { if (r <= 0 && w <= 0 && e <= 0) return (0); return (ENXIO); } r += pp->acr; w += pp->acw; e += pp->ace; if ((sc->flags & MD_READONLY) != 0 && w > 0) return (EROFS); if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) { sc->opencount = 1; } else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) { sc->opencount = 0; } return (0); } static void g_md_start(struct bio *bp) { struct md_s *sc; sc = bp->bio_to->geom->softc; if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE)) { mtx_lock(&sc->stat_mtx); devstat_start_transaction_bio(sc->devstat, bp); mtx_unlock(&sc->stat_mtx); } mtx_lock(&sc->queue_mtx); bioq_disksort(&sc->bio_queue, bp); wakeup(sc); mtx_unlock(&sc->queue_mtx); } #define MD_MALLOC_MOVE_ZERO 1 #define MD_MALLOC_MOVE_FILL 2 #define MD_MALLOC_MOVE_READ 3 #define MD_MALLOC_MOVE_WRITE 4 #define MD_MALLOC_MOVE_CMP 5 static int md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize, void *ptr, u_char fill, int op) { struct sf_buf *sf; vm_page_t m, *mp1; char *p, first; off_t *uc; unsigned n; int error, i, ma_offs1, sz, first_read; m = NULL; error = 0; sf = NULL; /* if (op == MD_MALLOC_MOVE_CMP) { gcc */ first = 0; first_read = 0; uc = ptr; mp1 = *mp; ma_offs1 = *ma_offs; /* } */ sched_pin(); for (n = sectorsize; n != 0; n -= sz) { sz = imin(PAGE_SIZE - *ma_offs, n); if (m != **mp) { if (sf != NULL) sf_buf_free(sf); m = **mp; sf = sf_buf_alloc(m, SFB_CPUPRIVATE | (md_malloc_wait ? 0 : SFB_NOWAIT)); if (sf == NULL) { error = ENOMEM; break; } } p = (char *)sf_buf_kva(sf) + *ma_offs; switch (op) { case MD_MALLOC_MOVE_ZERO: bzero(p, sz); break; case MD_MALLOC_MOVE_FILL: memset(p, fill, sz); break; case MD_MALLOC_MOVE_READ: bcopy(ptr, p, sz); cpu_flush_dcache(p, sz); break; case MD_MALLOC_MOVE_WRITE: bcopy(p, ptr, sz); break; case MD_MALLOC_MOVE_CMP: for (i = 0; i < sz; i++, p++) { if (!first_read) { *uc = (u_char)*p; first = *p; first_read = 1; } else if (*p != first) { error = EDOOFUS; break; } } break; default: KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op)); break; } if (error != 0) break; *ma_offs += sz; *ma_offs %= PAGE_SIZE; if (*ma_offs == 0) (*mp)++; ptr = (char *)ptr + sz; } if (sf != NULL) sf_buf_free(sf); sched_unpin(); if (op == MD_MALLOC_MOVE_CMP && error != 0) { *mp = mp1; *ma_offs = ma_offs1; } return (error); } static int md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs, unsigned len, void *ptr, u_char fill, int op) { bus_dma_segment_t *vlist; uint8_t *p, *end, first; off_t *uc; int ma_offs, seg_len; vlist = *pvlist; ma_offs = *pma_offs; uc = ptr; for (; len != 0; len -= seg_len) { seg_len = imin(vlist->ds_len - ma_offs, len); p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs; switch (op) { case MD_MALLOC_MOVE_ZERO: bzero(p, seg_len); break; case MD_MALLOC_MOVE_FILL: memset(p, fill, seg_len); break; case MD_MALLOC_MOVE_READ: bcopy(ptr, p, seg_len); cpu_flush_dcache(p, seg_len); break; case MD_MALLOC_MOVE_WRITE: bcopy(p, ptr, seg_len); break; case MD_MALLOC_MOVE_CMP: end = p + seg_len; first = *uc = *p; /* Confirm all following bytes match the first */ while (++p < end) { if (*p != first) return (EDOOFUS); } break; default: KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op)); break; } ma_offs += seg_len; if (ma_offs == vlist->ds_len) { ma_offs = 0; vlist++; } ptr = (uint8_t *)ptr + seg_len; } *pvlist = vlist; *pma_offs = ma_offs; return (0); } static int mdstart_malloc(struct md_s *sc, struct bio *bp) { u_char *dst; vm_page_t *m; bus_dma_segment_t *vlist; int i, error, error1, ma_offs, notmapped; off_t secno, nsec, uc; uintptr_t sp, osp; switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: case BIO_DELETE: break; default: return (EOPNOTSUPP); } notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0; vlist = (bp->bio_flags & BIO_VLIST) != 0 ? (bus_dma_segment_t *)bp->bio_data : NULL; if (notmapped) { m = bp->bio_ma; ma_offs = bp->bio_ma_offset; dst = NULL; KASSERT(vlist == NULL, ("vlists cannot be unmapped")); } else if (vlist != NULL) { ma_offs = bp->bio_ma_offset; dst = NULL; } else { dst = bp->bio_data; } nsec = bp->bio_length / sc->sectorsize; secno = bp->bio_offset / sc->sectorsize; error = 0; while (nsec--) { osp = s_read(sc->indir, secno); if (bp->bio_cmd == BIO_DELETE) { if (osp != 0) error = s_write(sc->indir, secno, 0); } else if (bp->bio_cmd == BIO_READ) { if (osp == 0) { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, 0, MD_MALLOC_MOVE_ZERO); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, NULL, 0, MD_MALLOC_MOVE_ZERO); } else bzero(dst, sc->sectorsize); } else if (osp <= 255) { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, osp, MD_MALLOC_MOVE_FILL); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, NULL, osp, MD_MALLOC_MOVE_FILL); } else memset(dst, osp, sc->sectorsize); } else { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_READ); } else if (vlist != NULL) { error = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_READ); } else { bcopy((void *)osp, dst, sc->sectorsize); cpu_flush_dcache(dst, sc->sectorsize); } } osp = 0; } else if (bp->bio_cmd == BIO_WRITE) { if (sc->flags & MD_COMPRESS) { if (notmapped) { error1 = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, &uc, 0, MD_MALLOC_MOVE_CMP); i = error1 == 0 ? sc->sectorsize : 0; } else if (vlist != NULL) { error1 = md_malloc_move_vlist(&vlist, &ma_offs, sc->sectorsize, &uc, 0, MD_MALLOC_MOVE_CMP); i = error1 == 0 ? sc->sectorsize : 0; } else { uc = dst[0]; for (i = 1; i < sc->sectorsize; i++) { if (dst[i] != uc) break; } } } else { i = 0; uc = 0; } if (i == sc->sectorsize) { if (osp != uc) error = s_write(sc->indir, secno, uc); } else { if (osp <= 255) { sp = (uintptr_t)uma_zalloc(sc->uma, md_malloc_wait ? M_WAITOK : M_NOWAIT); if (sp == 0) { error = ENOSPC; break; } if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)sp, 0, MD_MALLOC_MOVE_WRITE); } else if (vlist != NULL) { error = md_malloc_move_vlist( &vlist, &ma_offs, sc->sectorsize, (void *)sp, 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)sp, sc->sectorsize); } error = s_write(sc->indir, secno, sp); } else { if (notmapped) { error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_WRITE); } else if (vlist != NULL) { error = md_malloc_move_vlist( &vlist, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)osp, sc->sectorsize); } osp = 0; } } } else { error = EOPNOTSUPP; } if (osp > 255) uma_zfree(sc->uma, (void*)osp); if (error != 0) break; secno++; if (!notmapped && vlist == NULL) dst += sc->sectorsize; } bp->bio_resid = 0; return (error); } static void mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len) { off_t seg_len; while (offset >= vlist->ds_len) { offset -= vlist->ds_len; vlist++; } while (len != 0) { seg_len = omin(len, vlist->ds_len - offset); bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset), seg_len); offset = 0; src = (uint8_t *)src + seg_len; len -= seg_len; vlist++; } } static void mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len) { off_t seg_len; while (offset >= vlist->ds_len) { offset -= vlist->ds_len; vlist++; } while (len != 0) { seg_len = omin(len, vlist->ds_len - offset); bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst, seg_len); offset = 0; dst = (uint8_t *)dst + seg_len; len -= seg_len; vlist++; } } static int mdstart_preload(struct md_s *sc, struct bio *bp) { uint8_t *p; p = sc->pl_ptr + bp->bio_offset; switch (bp->bio_cmd) { case BIO_READ: if ((bp->bio_flags & BIO_VLIST) != 0) { mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data, bp->bio_ma_offset, bp->bio_length); } else { bcopy(p, bp->bio_data, bp->bio_length); } cpu_flush_dcache(bp->bio_data, bp->bio_length); break; case BIO_WRITE: if ((bp->bio_flags & BIO_VLIST) != 0) { mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data, bp->bio_ma_offset, p, bp->bio_length); } else { bcopy(bp->bio_data, p, bp->bio_length); } break; } bp->bio_resid = 0; return (0); } static int mdstart_vnode(struct md_s *sc, struct bio *bp) { int error; struct uio auio; struct iovec aiov; struct iovec *piov; struct mount *mp; struct vnode *vp; struct buf *pb; bus_dma_segment_t *vlist; struct thread *td; off_t iolen, iostart, len, zerosize; int ma_offs, npages; switch (bp->bio_cmd) { case BIO_READ: auio.uio_rw = UIO_READ; break; case BIO_WRITE: case BIO_DELETE: auio.uio_rw = UIO_WRITE; break; case BIO_FLUSH: break; default: return (EOPNOTSUPP); } td = curthread; vp = sc->vnode; pb = NULL; piov = NULL; ma_offs = bp->bio_ma_offset; len = bp->bio_length; /* * VNODE I/O * * If an error occurs, we set BIO_ERROR but we do not set * B_INVAL because (for a write anyway), the buffer is * still valid. */ if (bp->bio_cmd == BIO_FLUSH) { (void) vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_FSYNC(vp, MNT_WAIT, td); VOP_UNLOCK(vp, 0); vn_finished_write(mp); return (error); } auio.uio_offset = (vm_ooffset_t)bp->bio_offset; auio.uio_resid = bp->bio_length; auio.uio_segflg = UIO_SYSSPACE; auio.uio_td = td; if (bp->bio_cmd == BIO_DELETE) { /* * Emulate BIO_DELETE by writing zeros. */ zerosize = ZERO_REGION_SIZE - (ZERO_REGION_SIZE % sc->sectorsize); auio.uio_iovcnt = howmany(bp->bio_length, zerosize); piov = malloc(sizeof(*piov) * auio.uio_iovcnt, M_MD, M_WAITOK); auio.uio_iov = piov; while (len > 0) { piov->iov_base = __DECONST(void *, zero_region); piov->iov_len = len; if (len > zerosize) piov->iov_len = zerosize; len -= piov->iov_len; piov++; } piov = auio.uio_iov; } else if ((bp->bio_flags & BIO_VLIST) != 0) { piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK); auio.uio_iov = piov; vlist = (bus_dma_segment_t *)bp->bio_data; while (len > 0) { piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr + ma_offs); piov->iov_len = vlist->ds_len - ma_offs; if (piov->iov_len > len) piov->iov_len = len; len -= piov->iov_len; ma_offs = 0; vlist++; piov++; } auio.uio_iovcnt = piov - auio.uio_iov; piov = auio.uio_iov; } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pb = uma_zalloc(md_pbuf_zone, M_WAITOK); bp->bio_resid = len; unmapped_step: npages = atop(min(MAXPHYS, round_page(len + (ma_offs & PAGE_MASK)))); iolen = min(ptoa(npages) - (ma_offs & PAGE_MASK), len); KASSERT(iolen > 0, ("zero iolen")); pmap_qenter((vm_offset_t)pb->b_data, &bp->bio_ma[atop(ma_offs)], npages); aiov.iov_base = (void *)((vm_offset_t)pb->b_data + (ma_offs & PAGE_MASK)); aiov.iov_len = iolen; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_resid = iolen; } else { aiov.iov_base = bp->bio_data; aiov.iov_len = bp->bio_length; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; } iostart = auio.uio_offset; if (auio.uio_rw == UIO_READ) { vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(vp, &auio, 0, sc->cred); VOP_UNLOCK(vp, 0); } else { (void) vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_WRITE(vp, &auio, sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred); VOP_UNLOCK(vp, 0); vn_finished_write(mp); if (error == 0) sc->flags &= ~MD_VERIFY; } /* When MD_CACHE is set, try to avoid double-caching the data. */ if (error == 0 && (sc->flags & MD_CACHE) == 0) VOP_ADVISE(vp, iostart, auio.uio_offset - 1, POSIX_FADV_DONTNEED); if (pb != NULL) { pmap_qremove((vm_offset_t)pb->b_data, npages); if (error == 0) { len -= iolen; bp->bio_resid -= iolen; ma_offs += iolen; if (len > 0) goto unmapped_step; } uma_zfree(md_pbuf_zone, pb); } free(piov, M_MD); if (pb == NULL) bp->bio_resid = auio.uio_resid; return (error); } -static void -md_swap_page_free(vm_page_t m) -{ - - vm_page_xunbusy(m); - vm_page_free(m); -} - static int mdstart_swap(struct md_s *sc, struct bio *bp) { vm_page_t m; u_char *p; vm_pindex_t i, lastp; bus_dma_segment_t *vlist; int rv, ma_offs, offs, len, lastend; switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: case BIO_DELETE: break; default: return (EOPNOTSUPP); } p = bp->bio_data; ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ? bp->bio_ma_offset : 0; vlist = (bp->bio_flags & BIO_VLIST) != 0 ? (bus_dma_segment_t *)bp->bio_data : NULL; /* * offs is the offset at which to start operating on the * next (ie, first) page. lastp is the last page on * which we're going to operate. lastend is the ending * position within that last page (ie, PAGE_SIZE if * we're operating on complete aligned pages). */ offs = bp->bio_offset % PAGE_SIZE; lastp = (bp->bio_offset + bp->bio_length - 1) / PAGE_SIZE; lastend = (bp->bio_offset + bp->bio_length - 1) % PAGE_SIZE + 1; rv = VM_PAGER_OK; VM_OBJECT_WLOCK(sc->object); vm_object_pip_add(sc->object, 1); for (i = bp->bio_offset / PAGE_SIZE; i <= lastp; i++) { len = ((i == lastp) ? lastend : PAGE_SIZE) - offs; m = vm_page_grab(sc->object, i, VM_ALLOC_SYSTEM); if (bp->bio_cmd == BIO_READ) { if (vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); if (rv == VM_PAGER_ERROR) { - md_swap_page_free(m); + vm_page_free(m); break; } else if (rv == VM_PAGER_FAIL) { /* * Pager does not have the page. Zero * the allocated page, and mark it as * valid. Do not set dirty, the page * can be recreated if thrown out. */ pmap_zero_page(m); vm_page_valid(m); } if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(&m, offs, bp->bio_ma, ma_offs, len); } else if ((bp->bio_flags & BIO_VLIST) != 0) { physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs, vlist, ma_offs, len); cpu_flush_dcache(p, len); } else { physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len); cpu_flush_dcache(p, len); } } else if (bp->bio_cmd == BIO_WRITE) { if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); if (rv == VM_PAGER_ERROR) { - md_swap_page_free(m); + vm_page_free(m); break; } else if (rv == VM_PAGER_FAIL) pmap_zero_page(m); if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(bp->bio_ma, ma_offs, &m, offs, len); } else if ((bp->bio_flags & BIO_VLIST) != 0) { physcopyin_vlist(vlist, ma_offs, VM_PAGE_TO_PHYS(m) + offs, len); } else { physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); } vm_page_valid(m); if (m->dirty != VM_PAGE_BITS_ALL) { vm_page_dirty(m); vm_pager_page_unswapped(m); } } else if (bp->bio_cmd == BIO_DELETE) { if (len == PAGE_SIZE || vm_page_all_valid(m)) rv = VM_PAGER_OK; else rv = vm_pager_get_pages(sc->object, &m, 1, NULL, NULL); if (rv == VM_PAGER_ERROR) { - md_swap_page_free(m); + vm_page_free(m); break; } else if (rv == VM_PAGER_FAIL) { - md_swap_page_free(m); + vm_page_free(m); m = NULL; } else { /* Page is valid. */ if (len != PAGE_SIZE) { pmap_zero_page_area(m, offs, len); if (m->dirty != VM_PAGE_BITS_ALL) { vm_page_dirty(m); vm_pager_page_unswapped(m); } } else { vm_pager_page_unswapped(m); - md_swap_page_free(m); + vm_page_free(m); m = NULL; } } } if (m != NULL) { vm_page_xunbusy(m); vm_page_lock(m); if (vm_page_active(m)) vm_page_reference(m); else vm_page_activate(m); vm_page_unlock(m); } /* Actions on further pages start at offset 0 */ p += PAGE_SIZE - offs; offs = 0; ma_offs += len; } vm_object_pip_wakeup(sc->object); VM_OBJECT_WUNLOCK(sc->object); return (rv != VM_PAGER_ERROR ? 0 : ENOSPC); } static int mdstart_null(struct md_s *sc, struct bio *bp) { switch (bp->bio_cmd) { case BIO_READ: bzero(bp->bio_data, bp->bio_length); cpu_flush_dcache(bp->bio_data, bp->bio_length); break; case BIO_WRITE: break; } bp->bio_resid = 0; return (0); } static void md_kthread(void *arg) { struct md_s *sc; struct bio *bp; int error; sc = arg; thread_lock(curthread); sched_prio(curthread, PRIBIO); thread_unlock(curthread); if (sc->type == MD_VNODE) curthread->td_pflags |= TDP_NORUNNINGBUF; for (;;) { mtx_lock(&sc->queue_mtx); if (sc->flags & MD_SHUTDOWN) { sc->flags |= MD_EXITING; mtx_unlock(&sc->queue_mtx); kproc_exit(0); } bp = bioq_takefirst(&sc->bio_queue); if (!bp) { msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0); continue; } mtx_unlock(&sc->queue_mtx); if (bp->bio_cmd == BIO_GETATTR) { int isv = ((sc->flags & MD_VERIFY) != 0); if ((sc->fwsectors && sc->fwheads && (g_handleattr_int(bp, "GEOM::fwsectors", sc->fwsectors) || g_handleattr_int(bp, "GEOM::fwheads", sc->fwheads))) || g_handleattr_int(bp, "GEOM::candelete", 1)) error = -1; else if (sc->ident[0] != '\0' && g_handleattr_str(bp, "GEOM::ident", sc->ident)) error = -1; else if (g_handleattr_int(bp, "MNT::verified", isv)) error = -1; else error = EOPNOTSUPP; } else { error = sc->start(sc, bp); } if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { /* * Devstat uses (bio_bcount, bio_resid) for * determining the length of the completed part of * the i/o. g_io_deliver() will translate from * bio_completed to that, but it also destroys the * bio so we must do our own translation. */ bp->bio_bcount = bp->bio_length; bp->bio_resid = (error == -1 ? bp->bio_bcount : 0); devstat_end_transaction_bio(sc->devstat, bp); } if (error != -1) { bp->bio_completed = bp->bio_length; g_io_deliver(bp, error); } } } static struct md_s * mdfind(int unit) { struct md_s *sc; LIST_FOREACH(sc, &md_softc_list, list) { if (sc->unit == unit) break; } return (sc); } static struct md_s * mdnew(int unit, int *errp, enum md_types type) { struct md_s *sc; int error; *errp = 0; if (unit == -1) unit = alloc_unr(md_uh); else unit = alloc_unr_specific(md_uh, unit); if (unit == -1) { *errp = EBUSY; return (NULL); } sc = (struct md_s *)malloc(sizeof *sc, M_MD, M_WAITOK | M_ZERO); sc->type = type; bioq_init(&sc->bio_queue); mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF); mtx_init(&sc->stat_mtx, "md stat", NULL, MTX_DEF); sc->unit = unit; sprintf(sc->name, "md%d", unit); LIST_INSERT_HEAD(&md_softc_list, sc, list); error = kproc_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name); if (error == 0) return (sc); LIST_REMOVE(sc, list); mtx_destroy(&sc->stat_mtx); mtx_destroy(&sc->queue_mtx); free_unr(md_uh, sc->unit); free(sc, M_MD); *errp = error; return (NULL); } static void mdinit(struct md_s *sc) { struct g_geom *gp; struct g_provider *pp; g_topology_lock(); gp = g_new_geomf(&g_md_class, "md%d", sc->unit); gp->softc = sc; pp = g_new_providerf(gp, "md%d", sc->unit); pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; pp->mediasize = sc->mediasize; pp->sectorsize = sc->sectorsize; switch (sc->type) { case MD_MALLOC: case MD_VNODE: case MD_SWAP: pp->flags |= G_PF_ACCEPT_UNMAPPED; break; case MD_PRELOAD: case MD_NULL: break; } sc->gp = gp; sc->pp = pp; g_error_provider(pp, 0); g_topology_unlock(); sc->devstat = devstat_new_entry("md", sc->unit, sc->sectorsize, DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX); } static int mdcreate_malloc(struct md_s *sc, struct md_req *mdr) { uintptr_t sp; int error; off_t u; error = 0; if (mdr->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE)) return (EINVAL); if (mdr->md_sectorsize != 0 && !powerof2(mdr->md_sectorsize)) return (EINVAL); /* Compression doesn't make sense if we have reserved space */ if (mdr->md_options & MD_RESERVE) mdr->md_options &= ~MD_COMPRESS; if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; sc->flags = mdr->md_options & (MD_COMPRESS | MD_FORCE); sc->indir = dimension(sc->mediasize / sc->sectorsize); sc->uma = uma_zcreate(sc->name, sc->sectorsize, NULL, NULL, NULL, NULL, 0x1ff, 0); if (mdr->md_options & MD_RESERVE) { off_t nsectors; nsectors = sc->mediasize / sc->sectorsize; for (u = 0; u < nsectors; u++) { sp = (uintptr_t)uma_zalloc(sc->uma, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); if (sp != 0) error = s_write(sc->indir, u, sp); else error = ENOMEM; if (error != 0) break; } } return (error); } static int mdsetcred(struct md_s *sc, struct ucred *cred) { char *tmpbuf; int error = 0; /* * Set credits in our softc */ if (sc->cred) crfree(sc->cred); sc->cred = crhold(cred); /* * Horrible kludge to establish credentials for NFS XXX. */ if (sc->vnode) { struct uio auio; struct iovec aiov; tmpbuf = malloc(sc->sectorsize, M_TEMP, M_WAITOK); bzero(&auio, sizeof(auio)); aiov.iov_base = tmpbuf; aiov.iov_len = sc->sectorsize; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = 0; auio.uio_rw = UIO_READ; auio.uio_segflg = UIO_SYSSPACE; auio.uio_resid = aiov.iov_len; vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(sc->vnode, &auio, 0, sc->cred); VOP_UNLOCK(sc->vnode, 0); free(tmpbuf, M_TEMP); } return (error); } static int mdcreate_vnode(struct md_s *sc, struct md_req *mdr, struct thread *td) { struct vattr vattr; struct nameidata nd; char *fname; int error, flags; fname = mdr->md_file; if (mdr->md_file_seg == UIO_USERSPACE) { error = copyinstr(fname, sc->file, sizeof(sc->file), NULL); if (error != 0) return (error); } else if (mdr->md_file_seg == UIO_SYSSPACE) strlcpy(sc->file, fname, sizeof(sc->file)); else return (EDOOFUS); /* * If the user specified that this is a read only device, don't * set the FWRITE mask before trying to open the backing store. */ flags = FREAD | ((mdr->md_options & MD_READONLY) ? 0 : FWRITE) \ | ((mdr->md_options & MD_VERIFY) ? O_VERIFY : 0); NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->file, td); error = vn_open(&nd, &flags, 0, NULL); if (error != 0) return (error); NDFREE(&nd, NDF_ONLY_PNBUF); if (nd.ni_vp->v_type != VREG) { error = EINVAL; goto bad; } error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); if (error != 0) goto bad; if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); if (nd.ni_vp->v_iflag & VI_DOOMED) { /* Forced unmount. */ error = EBADF; goto bad; } } nd.ni_vp->v_vflag |= VV_MD; VOP_UNLOCK(nd.ni_vp, 0); if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; snprintf(sc->ident, sizeof(sc->ident), "MD-DEV%ju-INO%ju", (uintmax_t)vattr.va_fsid, (uintmax_t)vattr.va_fileid); sc->flags = mdr->md_options & (MD_ASYNC | MD_CACHE | MD_FORCE | MD_VERIFY); if (!(flags & FWRITE)) sc->flags |= MD_READONLY; sc->vnode = nd.ni_vp; error = mdsetcred(sc, td->td_ucred); if (error != 0) { sc->vnode = NULL; vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); nd.ni_vp->v_vflag &= ~VV_MD; goto bad; } return (0); bad: VOP_UNLOCK(nd.ni_vp, 0); (void)vn_close(nd.ni_vp, flags, td->td_ucred, td); return (error); } static void g_md_providergone(struct g_provider *pp) { struct md_s *sc = pp->geom->softc; mtx_lock(&sc->queue_mtx); sc->flags |= MD_PROVIDERGONE; wakeup(&sc->flags); mtx_unlock(&sc->queue_mtx); } static int mddestroy(struct md_s *sc, struct thread *td) { if (sc->gp) { g_topology_lock(); g_wither_geom(sc->gp, ENXIO); g_topology_unlock(); mtx_lock(&sc->queue_mtx); while (!(sc->flags & MD_PROVIDERGONE)) msleep(&sc->flags, &sc->queue_mtx, PRIBIO, "mddestroy", 0); mtx_unlock(&sc->queue_mtx); } if (sc->devstat) { devstat_remove_entry(sc->devstat); sc->devstat = NULL; } mtx_lock(&sc->queue_mtx); sc->flags |= MD_SHUTDOWN; wakeup(sc); while (!(sc->flags & MD_EXITING)) msleep(sc->procp, &sc->queue_mtx, PRIBIO, "mddestroy", hz / 10); mtx_unlock(&sc->queue_mtx); mtx_destroy(&sc->stat_mtx); mtx_destroy(&sc->queue_mtx); if (sc->vnode != NULL) { vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); sc->vnode->v_vflag &= ~VV_MD; VOP_UNLOCK(sc->vnode, 0); (void)vn_close(sc->vnode, sc->flags & MD_READONLY ? FREAD : (FREAD|FWRITE), sc->cred, td); } if (sc->cred != NULL) crfree(sc->cred); if (sc->object != NULL) vm_object_deallocate(sc->object); if (sc->indir) destroy_indir(sc, sc->indir); if (sc->uma) uma_zdestroy(sc->uma); LIST_REMOVE(sc, list); free_unr(md_uh, sc->unit); free(sc, M_MD); return (0); } static int mdresize(struct md_s *sc, struct md_req *mdr) { int error, res; vm_pindex_t oldpages, newpages; switch (sc->type) { case MD_VNODE: case MD_NULL: break; case MD_SWAP: if (mdr->md_mediasize <= 0 || (mdr->md_mediasize % PAGE_SIZE) != 0) return (EDOM); oldpages = OFF_TO_IDX(round_page(sc->mediasize)); newpages = OFF_TO_IDX(round_page(mdr->md_mediasize)); if (newpages < oldpages) { VM_OBJECT_WLOCK(sc->object); vm_object_page_remove(sc->object, newpages, 0, 0); swap_pager_freespace(sc->object, newpages, oldpages - newpages); swap_release_by_cred(IDX_TO_OFF(oldpages - newpages), sc->cred); sc->object->charge = IDX_TO_OFF(newpages); sc->object->size = newpages; VM_OBJECT_WUNLOCK(sc->object); } else if (newpages > oldpages) { res = swap_reserve_by_cred(IDX_TO_OFF(newpages - oldpages), sc->cred); if (!res) return (ENOMEM); if ((mdr->md_options & MD_RESERVE) || (sc->flags & MD_RESERVE)) { error = swap_pager_reserve(sc->object, oldpages, newpages - oldpages); if (error < 0) { swap_release_by_cred( IDX_TO_OFF(newpages - oldpages), sc->cred); return (EDOM); } } VM_OBJECT_WLOCK(sc->object); sc->object->charge = IDX_TO_OFF(newpages); sc->object->size = newpages; VM_OBJECT_WUNLOCK(sc->object); } break; default: return (EOPNOTSUPP); } sc->mediasize = mdr->md_mediasize; g_topology_lock(); g_resize_provider(sc->pp, sc->mediasize); g_topology_unlock(); return (0); } static int mdcreate_swap(struct md_s *sc, struct md_req *mdr, struct thread *td) { vm_ooffset_t npage; int error; /* * Range check. Disallow negative sizes and sizes not being * multiple of page size. */ if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) return (EDOM); /* * Allocate an OBJT_SWAP object. * * Note the truncation. */ if ((mdr->md_options & MD_VERIFY) != 0) return (EINVAL); npage = mdr->md_mediasize / PAGE_SIZE; if (mdr->md_fwsectors != 0) sc->fwsectors = mdr->md_fwsectors; if (mdr->md_fwheads != 0) sc->fwheads = mdr->md_fwheads; sc->object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * npage, VM_PROT_DEFAULT, 0, td->td_ucred); if (sc->object == NULL) return (ENOMEM); sc->flags = mdr->md_options & (MD_FORCE | MD_RESERVE); if (mdr->md_options & MD_RESERVE) { if (swap_pager_reserve(sc->object, 0, npage) < 0) { error = EDOM; goto finish; } } error = mdsetcred(sc, td->td_ucred); finish: if (error != 0) { vm_object_deallocate(sc->object); sc->object = NULL; } return (error); } static int mdcreate_null(struct md_s *sc, struct md_req *mdr, struct thread *td) { /* * Range check. Disallow negative sizes and sizes not being * multiple of page size. */ if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) return (EDOM); return (0); } static int kern_mdattach_locked(struct thread *td, struct md_req *mdr) { struct md_s *sc; unsigned sectsize; int error, i; sx_assert(&md_sx, SA_XLOCKED); switch (mdr->md_type) { case MD_MALLOC: case MD_PRELOAD: case MD_VNODE: case MD_SWAP: case MD_NULL: break; default: return (EINVAL); } if (mdr->md_sectorsize == 0) sectsize = DEV_BSIZE; else sectsize = mdr->md_sectorsize; if (sectsize > MAXPHYS || mdr->md_mediasize < sectsize) return (EINVAL); if (mdr->md_options & MD_AUTOUNIT) sc = mdnew(-1, &error, mdr->md_type); else { if (mdr->md_unit > INT_MAX) return (EINVAL); sc = mdnew(mdr->md_unit, &error, mdr->md_type); } if (sc == NULL) return (error); if (mdr->md_label != NULL) error = copyinstr(mdr->md_label, sc->label, sizeof(sc->label), NULL); if (error != 0) goto err_after_new; if (mdr->md_options & MD_AUTOUNIT) mdr->md_unit = sc->unit; sc->mediasize = mdr->md_mediasize; sc->sectorsize = sectsize; error = EDOOFUS; switch (sc->type) { case MD_MALLOC: sc->start = mdstart_malloc; error = mdcreate_malloc(sc, mdr); break; case MD_PRELOAD: /* * We disallow attaching preloaded memory disks via * ioctl. Preloaded memory disks are automatically * attached in g_md_init(). */ error = EOPNOTSUPP; break; case MD_VNODE: sc->start = mdstart_vnode; error = mdcreate_vnode(sc, mdr, td); break; case MD_SWAP: sc->start = mdstart_swap; error = mdcreate_swap(sc, mdr, td); break; case MD_NULL: sc->start = mdstart_null; error = mdcreate_null(sc, mdr, td); break; } err_after_new: if (error != 0) { mddestroy(sc, td); return (error); } /* Prune off any residual fractional sector */ i = sc->mediasize % sc->sectorsize; sc->mediasize -= i; mdinit(sc); return (0); } static int kern_mdattach(struct thread *td, struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdattach_locked(td, mdr); sx_xunlock(&md_sx); return (error); } static int kern_mddetach_locked(struct thread *td, struct md_req *mdr) { struct md_s *sc; sx_assert(&md_sx, SA_XLOCKED); if (mdr->md_mediasize != 0 || (mdr->md_options & ~MD_FORCE) != 0) return (EINVAL); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); if (sc->opencount != 0 && !(sc->flags & MD_FORCE) && !(mdr->md_options & MD_FORCE)) return (EBUSY); return (mddestroy(sc, td)); } static int kern_mddetach(struct thread *td, struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mddetach_locked(td, mdr); sx_xunlock(&md_sx); return (error); } static int kern_mdresize_locked(struct md_req *mdr) { struct md_s *sc; sx_assert(&md_sx, SA_XLOCKED); if ((mdr->md_options & ~(MD_FORCE | MD_RESERVE)) != 0) return (EINVAL); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); if (mdr->md_mediasize < sc->sectorsize) return (EINVAL); if (mdr->md_mediasize < sc->mediasize && !(sc->flags & MD_FORCE) && !(mdr->md_options & MD_FORCE)) return (EBUSY); return (mdresize(sc, mdr)); } static int kern_mdresize(struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdresize_locked(mdr); sx_xunlock(&md_sx); return (error); } static int kern_mdquery_locked(struct md_req *mdr) { struct md_s *sc; int error; sx_assert(&md_sx, SA_XLOCKED); sc = mdfind(mdr->md_unit); if (sc == NULL) return (ENOENT); mdr->md_type = sc->type; mdr->md_options = sc->flags; mdr->md_mediasize = sc->mediasize; mdr->md_sectorsize = sc->sectorsize; error = 0; if (mdr->md_label != NULL) { error = copyout(sc->label, mdr->md_label, strlen(sc->label) + 1); if (error != 0) return (error); } if (sc->type == MD_VNODE || (sc->type == MD_PRELOAD && mdr->md_file != NULL)) error = copyout(sc->file, mdr->md_file, strlen(sc->file) + 1); return (error); } static int kern_mdquery(struct md_req *mdr) { int error; sx_xlock(&md_sx); error = kern_mdquery_locked(mdr); sx_xunlock(&md_sx); return (error); } /* Copy members that are not userspace pointers. */ #define MD_IOCTL2REQ(mdio, mdr) do { \ (mdr)->md_unit = (mdio)->md_unit; \ (mdr)->md_type = (mdio)->md_type; \ (mdr)->md_mediasize = (mdio)->md_mediasize; \ (mdr)->md_sectorsize = (mdio)->md_sectorsize; \ (mdr)->md_options = (mdio)->md_options; \ (mdr)->md_fwheads = (mdio)->md_fwheads; \ (mdr)->md_fwsectors = (mdio)->md_fwsectors; \ (mdr)->md_units = &(mdio)->md_pad[0]; \ (mdr)->md_units_nitems = nitems((mdio)->md_pad); \ } while(0) /* Copy members that might have been updated */ #define MD_REQ2IOCTL(mdr, mdio) do { \ (mdio)->md_unit = (mdr)->md_unit; \ (mdio)->md_type = (mdr)->md_type; \ (mdio)->md_mediasize = (mdr)->md_mediasize; \ (mdio)->md_sectorsize = (mdr)->md_sectorsize; \ (mdio)->md_options = (mdr)->md_options; \ (mdio)->md_fwheads = (mdr)->md_fwheads; \ (mdio)->md_fwsectors = (mdr)->md_fwsectors; \ } while(0) static int mdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { struct md_req mdr; int error; if (md_debug) printf("mdctlioctl(%s %lx %p %x %p)\n", devtoname(dev), cmd, addr, flags, td); bzero(&mdr, sizeof(mdr)); switch (cmd) { case MDIOCATTACH: case MDIOCDETACH: case MDIOCRESIZE: case MDIOCQUERY: { struct md_ioctl *mdio = (struct md_ioctl *)addr; if (mdio->md_version != MDIOVERSION) return (EINVAL); MD_IOCTL2REQ(mdio, &mdr); mdr.md_file = mdio->md_file; mdr.md_file_seg = UIO_USERSPACE; /* If the file is adjacent to the md_ioctl it's in kernel. */ if ((void *)mdio->md_file == (void *)(mdio + 1)) mdr.md_file_seg = UIO_SYSSPACE; mdr.md_label = mdio->md_label; break; } #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: case MDIOCDETACH_32: case MDIOCRESIZE_32: case MDIOCQUERY_32: { struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; if (mdio->md_version != MDIOVERSION) return (EINVAL); MD_IOCTL2REQ(mdio, &mdr); mdr.md_file = (void *)(uintptr_t)mdio->md_file; mdr.md_file_seg = UIO_USERSPACE; mdr.md_label = (void *)(uintptr_t)mdio->md_label; break; } #endif default: /* Fall through to handler switch. */ break; } error = 0; switch (cmd) { case MDIOCATTACH: #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: #endif error = kern_mdattach(td, &mdr); break; case MDIOCDETACH: #ifdef COMPAT_FREEBSD32 case MDIOCDETACH_32: #endif error = kern_mddetach(td, &mdr); break; case MDIOCRESIZE: #ifdef COMPAT_FREEBSD32 case MDIOCRESIZE_32: #endif error = kern_mdresize(&mdr); break; case MDIOCQUERY: #ifdef COMPAT_FREEBSD32 case MDIOCQUERY_32: #endif error = kern_mdquery(&mdr); break; default: error = ENOIOCTL; } switch (cmd) { case MDIOCATTACH: case MDIOCQUERY: { struct md_ioctl *mdio = (struct md_ioctl *)addr; MD_REQ2IOCTL(&mdr, mdio); break; } #ifdef COMPAT_FREEBSD32 case MDIOCATTACH_32: case MDIOCQUERY_32: { struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; MD_REQ2IOCTL(&mdr, mdio); break; } #endif default: /* Other commands to not alter mdr. */ break; } return (error); } static void md_preloaded(u_char *image, size_t length, const char *name) { struct md_s *sc; int error; sc = mdnew(-1, &error, MD_PRELOAD); if (sc == NULL) return; sc->mediasize = length; sc->sectorsize = DEV_BSIZE; sc->pl_ptr = image; sc->pl_len = length; sc->start = mdstart_preload; if (name != NULL) strlcpy(sc->file, name, sizeof(sc->file)); #ifdef MD_ROOT if (sc->unit == 0) { #ifndef ROOTDEVNAME rootdevnames[0] = MD_ROOT_FSTYPE ":/dev/md0"; #endif #ifdef MD_ROOT_READONLY sc->flags |= MD_READONLY; #endif } #endif mdinit(sc); if (name != NULL) { printf("%s%d: Preloaded image <%s> %zd bytes at %p\n", MD_NAME, sc->unit, name, length, image); } else { printf("%s%d: Embedded image %zd bytes at %p\n", MD_NAME, sc->unit, length, image); } } static void g_md_init(struct g_class *mp __unused) { caddr_t mod; u_char *ptr, *name, *type; unsigned len; int i; /* figure out log2(NINDIR) */ for (i = NINDIR, nshift = -1; i; nshift++) i >>= 1; mod = NULL; sx_init(&md_sx, "MD config lock"); g_topology_unlock(); md_uh = new_unrhdr(0, INT_MAX, NULL); #ifdef MD_ROOT if (mfs_root_size != 0) { sx_xlock(&md_sx); #ifdef MD_ROOT_MEM md_preloaded(mfs_root, mfs_root_size, NULL); #else md_preloaded(__DEVOLATILE(u_char *, &mfs_root), mfs_root_size, NULL); #endif sx_xunlock(&md_sx); } #endif /* XXX: are preload_* static or do they need Giant ? */ while ((mod = preload_search_next_name(mod)) != NULL) { name = (char *)preload_search_info(mod, MODINFO_NAME); if (name == NULL) continue; type = (char *)preload_search_info(mod, MODINFO_TYPE); if (type == NULL) continue; if (strcmp(type, "md_image") && strcmp(type, "mfs_root")) continue; ptr = preload_fetch_addr(mod); len = preload_fetch_size(mod); if (ptr != NULL && len != 0) { sx_xlock(&md_sx); md_preloaded(ptr, len, name); sx_xunlock(&md_sx); } } md_pbuf_zone = pbuf_zsecond_create("mdpbuf", nswbuf / 10); status_dev = make_dev(&mdctl_cdevsw, INT_MAX, UID_ROOT, GID_WHEEL, 0600, MDCTL_NAME); g_topology_lock(); } static void g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp) { struct md_s *mp; char *type; mp = gp->softc; if (mp == NULL) return; switch (mp->type) { case MD_MALLOC: type = "malloc"; break; case MD_PRELOAD: type = "preload"; break; case MD_VNODE: type = "vnode"; break; case MD_SWAP: type = "swap"; break; case MD_NULL: type = "null"; break; default: type = "unknown"; break; } if (pp != NULL) { if (indent == NULL) { sbuf_printf(sb, " u %d", mp->unit); sbuf_printf(sb, " s %ju", (uintmax_t) mp->sectorsize); sbuf_printf(sb, " f %ju", (uintmax_t) mp->fwheads); sbuf_printf(sb, " fs %ju", (uintmax_t) mp->fwsectors); sbuf_printf(sb, " l %ju", (uintmax_t) mp->mediasize); sbuf_printf(sb, " t %s", type); if ((mp->type == MD_VNODE && mp->vnode != NULL) || (mp->type == MD_PRELOAD && mp->file[0] != '\0')) sbuf_printf(sb, " file %s", mp->file); sbuf_printf(sb, " label %s", mp->label); } else { sbuf_printf(sb, "%s%d\n", indent, mp->unit); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->sectorsize); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->fwheads); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->fwsectors); if (mp->ident[0] != '\0') { sbuf_printf(sb, "%s", indent); g_conf_printf_escaped(sb, "%s", mp->ident); sbuf_printf(sb, "\n"); } sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t) mp->mediasize); sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_COMPRESS) == 0 ? "off": "on"); sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_READONLY) == 0 ? "read-write": "read-only"); sbuf_printf(sb, "%s%s\n", indent, type); if ((mp->type == MD_VNODE && mp->vnode != NULL) || (mp->type == MD_PRELOAD && mp->file[0] != '\0')) { sbuf_printf(sb, "%s", indent); g_conf_printf_escaped(sb, "%s", mp->file); sbuf_printf(sb, "\n"); } if (mp->type == MD_VNODE) sbuf_printf(sb, "%s%s\n", indent, (mp->flags & MD_CACHE) == 0 ? "off": "on"); sbuf_printf(sb, "%s\n"); } } } static void g_md_fini(struct g_class *mp __unused) { sx_destroy(&md_sx); if (status_dev != NULL) destroy_dev(status_dev); uma_zdestroy(md_pbuf_zone); delete_unrhdr(md_uh); } Index: head/sys/vm/vm_glue.c =================================================================== --- head/sys/vm/vm_glue.c (revision 355313) +++ head/sys/vm/vm_glue.c (revision 355314) @@ -1,603 +1,604 @@ /*- * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU) * * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vm_glue.c 8.6 (Berkeley) 1/5/94 * * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include "opt_kstack_pages.h" #include "opt_kstack_max_pages.h" #include "opt_kstack_usage_prof.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * MPSAFE * * WARNING! This code calls vm_map_check_protection() which only checks * the associated vm_map_entry range. It does not determine whether the * contents of the memory is actually readable or writable. In most cases * just checking the vm_map_entry is sufficient within the kernel's address * space. */ int kernacc(void *addr, int len, int rw) { boolean_t rv; vm_offset_t saddr, eaddr; vm_prot_t prot; KASSERT((rw & ~VM_PROT_ALL) == 0, ("illegal ``rw'' argument to kernacc (%x)\n", rw)); if ((vm_offset_t)addr + len > vm_map_max(kernel_map) || (vm_offset_t)addr + len < (vm_offset_t)addr) return (FALSE); prot = rw; saddr = trunc_page((vm_offset_t)addr); eaddr = round_page((vm_offset_t)addr + len); vm_map_lock_read(kernel_map); rv = vm_map_check_protection(kernel_map, saddr, eaddr, prot); vm_map_unlock_read(kernel_map); return (rv == TRUE); } /* * MPSAFE * * WARNING! This code calls vm_map_check_protection() which only checks * the associated vm_map_entry range. It does not determine whether the * contents of the memory is actually readable or writable. vmapbuf(), * vm_fault_quick(), or copyin()/copout()/su*()/fu*() functions should be * used in conjunction with this call. */ int useracc(void *addr, int len, int rw) { boolean_t rv; vm_prot_t prot; vm_map_t map; KASSERT((rw & ~VM_PROT_ALL) == 0, ("illegal ``rw'' argument to useracc (%x)\n", rw)); prot = rw; map = &curproc->p_vmspace->vm_map; if ((vm_offset_t)addr + len > vm_map_max(map) || (vm_offset_t)addr + len < (vm_offset_t)addr) { return (FALSE); } vm_map_lock_read(map); rv = vm_map_check_protection(map, trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len), prot); vm_map_unlock_read(map); return (rv == TRUE); } int vslock(void *addr, size_t len) { vm_offset_t end, last, start; vm_size_t npages; int error; last = (vm_offset_t)addr + len; start = trunc_page((vm_offset_t)addr); end = round_page(last); if (last < (vm_offset_t)addr || end < (vm_offset_t)addr) return (EINVAL); npages = atop(end - start); if (npages > vm_page_max_user_wired) return (ENOMEM); error = vm_map_wire(&curproc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); if (error == KERN_SUCCESS) { curthread->td_vslock_sz += len; return (0); } /* * Return EFAULT on error to match copy{in,out}() behaviour * rather than returning ENOMEM like mlock() would. */ return (EFAULT); } void vsunlock(void *addr, size_t len) { /* Rely on the parameter sanity checks performed by vslock(). */ MPASS(curthread->td_vslock_sz >= len); curthread->td_vslock_sz -= len; (void)vm_map_unwire(&curproc->p_vmspace->vm_map, trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len), VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); } /* * Pin the page contained within the given object at the given offset. If the * page is not resident, allocate and load it using the given object's pager. * Return the pinned page if successful; otherwise, return NULL. */ static vm_page_t vm_imgact_hold_page(vm_object_t object, vm_ooffset_t offset) { vm_page_t m; vm_pindex_t pindex; pindex = OFF_TO_IDX(offset); VM_OBJECT_WLOCK(object); (void)vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); VM_OBJECT_WUNLOCK(object); return (m); } /* * Return a CPU private mapping to the page at the given offset within the * given object. The page is pinned before it is mapped. */ struct sf_buf * vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset) { vm_page_t m; m = vm_imgact_hold_page(object, offset); if (m == NULL) return (NULL); sched_pin(); return (sf_buf_alloc(m, SFB_CPUPRIVATE)); } /* * Destroy the given CPU private mapping and unpin the page that it mapped. */ void vm_imgact_unmap_page(struct sf_buf *sf) { vm_page_t m; m = sf_buf_page(sf); sf_buf_free(sf); sched_unpin(); vm_page_unwire(m, PQ_ACTIVE); } void vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset_t sz) { pmap_sync_icache(map->pmap, va, sz); } static uma_zone_t kstack_cache; static int kstack_cache_size; static int kstack_domain_iter; static int sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS) { int error, newsize; newsize = kstack_cache_size; error = sysctl_handle_int(oidp, &newsize, 0, req); if (error == 0 && req->newptr && newsize != kstack_cache_size) uma_zone_set_maxcache(kstack_cache, newsize); return (error); } SYSCTL_PROC(_vm, OID_AUTO, kstack_cache_size, CTLTYPE_INT|CTLFLAG_RW, &kstack_cache_size, 0, sysctl_kstack_cache_size, "IU", "Maximum number of cached kernel stacks"); /* * Create the kernel stack (including pcb for i386) for a new thread. * This routine directly affects the fork perf for a process and * create performance for a thread. */ static vm_offset_t vm_thread_stack_create(struct domainset *ds, vm_object_t *ksobjp, int pages) { vm_page_t ma[KSTACK_MAX_PAGES]; vm_object_t ksobj; vm_offset_t ks; int i; /* * Allocate an object for the kstack. */ ksobj = vm_object_allocate(OBJT_DEFAULT, pages); /* * Get a kernel virtual address for this thread's kstack. */ #if defined(__mips__) /* * We need to align the kstack's mapped address to fit within * a single TLB entry. */ if (vmem_xalloc(kernel_arena, (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE, PAGE_SIZE * 2, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, M_BESTFIT | M_NOWAIT, &ks)) { ks = 0; } #else ks = kva_alloc((pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); #endif if (ks == 0) { printf("%s: kstack allocation failed\n", __func__); vm_object_deallocate(ksobj); return (0); } if (vm_ndomains > 1) { ksobj->domain.dr_policy = ds; ksobj->domain.dr_iter = atomic_fetchadd_int(&kstack_domain_iter, 1); } if (KSTACK_GUARD_PAGES != 0) { pmap_qremove(ks, KSTACK_GUARD_PAGES); ks += KSTACK_GUARD_PAGES * PAGE_SIZE; } /* * For the length of the stack, link in a real page of ram for each * page of stack. */ VM_OBJECT_WLOCK(ksobj); (void)vm_page_grab_pages(ksobj, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED, ma, pages); for (i = 0; i < pages; i++) ma[i]->valid = VM_PAGE_BITS_ALL; VM_OBJECT_WUNLOCK(ksobj); pmap_qenter(ks, ma, pages); *ksobjp = ksobj; return (ks); } static void vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) { vm_page_t m; int i; pmap_qremove(ks, pages); VM_OBJECT_WLOCK(ksobj); for (i = 0; i < pages; i++) { m = vm_page_lookup(ksobj, i); if (m == NULL) panic("%s: kstack already missing?", __func__); + vm_page_busy_acquire(m, 0); vm_page_unwire_noq(m); vm_page_free(m); } VM_OBJECT_WUNLOCK(ksobj); vm_object_deallocate(ksobj); kva_free(ks - (KSTACK_GUARD_PAGES * PAGE_SIZE), (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); } /* * Allocate the kernel stack for a new thread. */ int vm_thread_new(struct thread *td, int pages) { vm_object_t ksobj; vm_offset_t ks; /* Bounds check */ if (pages <= 1) pages = kstack_pages; else if (pages > KSTACK_MAX_PAGES) pages = KSTACK_MAX_PAGES; ks = 0; ksobj = NULL; if (pages == kstack_pages && kstack_cache != NULL) { ks = (vm_offset_t)uma_zalloc(kstack_cache, M_NOWAIT); if (ks != 0) ksobj = PHYS_TO_VM_PAGE(pmap_kextract(ks))->object; } /* * Ensure that kstack objects can draw pages from any memory * domain. Otherwise a local memory shortage can block a process * swap-in. */ if (ks == 0) ks = vm_thread_stack_create(DOMAINSET_PREF(PCPU_GET(domain)), &ksobj, pages); if (ks == 0) return (0); td->td_kstack_obj = ksobj; td->td_kstack = ks; td->td_kstack_pages = pages; return (1); } /* * Dispose of a thread's kernel stack. */ void vm_thread_dispose(struct thread *td) { vm_object_t ksobj; vm_offset_t ks; int pages; pages = td->td_kstack_pages; ksobj = td->td_kstack_obj; ks = td->td_kstack; td->td_kstack = 0; td->td_kstack_pages = 0; if (pages == kstack_pages) uma_zfree(kstack_cache, (void *)ks); else vm_thread_stack_dispose(ksobj, ks, pages); } static int kstack_import(void *arg, void **store, int cnt, int domain, int flags) { struct domainset *ds; vm_object_t ksobj; int i; if (domain == UMA_ANYDOMAIN) ds = DOMAINSET_RR(); else ds = DOMAINSET_PREF(domain); for (i = 0; i < cnt; i++) { store[i] = (void *)vm_thread_stack_create(ds, &ksobj, kstack_pages); if (store[i] == NULL) break; } return (i); } static void kstack_release(void *arg, void **store, int cnt) { vm_offset_t ks; int i; for (i = 0; i < cnt; i++) { ks = (vm_offset_t)store[i]; vm_thread_stack_dispose( PHYS_TO_VM_PAGE(pmap_kextract(ks))->object, ks, kstack_pages); } } static void kstack_cache_init(void *null) { kstack_cache = uma_zcache_create("kstack_cache", kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL, kstack_import, kstack_release, NULL, UMA_ZONE_NUMA); kstack_cache_size = imax(128, mp_ncpus * 4); uma_zone_set_maxcache(kstack_cache, kstack_cache_size); } SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL); #ifdef KSTACK_USAGE_PROF /* * Track maximum stack used by a thread in kernel. */ static int max_kstack_used; SYSCTL_INT(_debug, OID_AUTO, max_kstack_used, CTLFLAG_RD, &max_kstack_used, 0, "Maxiumum stack depth used by a thread in kernel"); void intr_prof_stack_use(struct thread *td, struct trapframe *frame) { vm_offset_t stack_top; vm_offset_t current; int used, prev_used; /* * Testing for interrupted kernel mode isn't strictly * needed. It optimizes the execution, since interrupts from * usermode will have only the trap frame on the stack. */ if (TRAPF_USERMODE(frame)) return; stack_top = td->td_kstack + td->td_kstack_pages * PAGE_SIZE; current = (vm_offset_t)(uintptr_t)&stack_top; /* * Try to detect if interrupt is using kernel thread stack. * Hardware could use a dedicated stack for interrupt handling. */ if (stack_top <= current || current < td->td_kstack) return; used = stack_top - current; for (;;) { prev_used = max_kstack_used; if (prev_used >= used) break; if (atomic_cmpset_int(&max_kstack_used, prev_used, used)) break; } } #endif /* KSTACK_USAGE_PROF */ /* * Implement fork's actions on an address space. * Here we arrange for the address space to be copied or referenced, * allocate a user struct (pcb and kernel stack), then call the * machine-dependent layer to fill those in and make the new process * ready to run. The new process is set up so that it returns directly * to user mode to avoid stack copying and relocation problems. */ int vm_forkproc(struct thread *td, struct proc *p2, struct thread *td2, struct vmspace *vm2, int flags) { struct proc *p1 = td->td_proc; struct domainset *dset; int error; if ((flags & RFPROC) == 0) { /* * Divorce the memory, if it is shared, essentially * this changes shared memory amongst threads, into * COW locally. */ if ((flags & RFMEM) == 0) { if (p1->p_vmspace->vm_refcnt > 1) { error = vmspace_unshare(p1); if (error) return (error); } } cpu_fork(td, p2, td2, flags); return (0); } if (flags & RFMEM) { p2->p_vmspace = p1->p_vmspace; atomic_add_int(&p1->p_vmspace->vm_refcnt, 1); } dset = td2->td_domain.dr_policy; while (vm_page_count_severe_set(&dset->ds_mask)) { vm_wait_doms(&dset->ds_mask); } if ((flags & RFMEM) == 0) { p2->p_vmspace = vm2; if (p1->p_vmspace->vm_shm) shmfork(p1, p2); } /* * cpu_fork will copy and update the pcb, set up the kernel stack, * and make the child ready to run. */ cpu_fork(td, p2, td2, flags); return (0); } /* * Called after process has been wait(2)'ed upon and is being reaped. * The idea is to reclaim resources that we could not reclaim while * the process was still executing. */ void vm_waitproc(p) struct proc *p; { vmspace_exitfree(p); /* and clean-out the vmspace */ } void kick_proc0(void) { wakeup(&proc0); } Index: head/sys/x86/iommu/intel_utils.c =================================================================== --- head/sys/x86/iommu/intel_utils.c (revision 355313) +++ head/sys/x86/iommu/intel_utils.c (revision 355314) @@ -1,674 +1,674 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include u_int dmar_nd2mask(u_int nd) { static const u_int masks[] = { 0x000f, /* nd == 0 */ 0x002f, /* nd == 1 */ 0x00ff, /* nd == 2 */ 0x02ff, /* nd == 3 */ 0x0fff, /* nd == 4 */ 0x2fff, /* nd == 5 */ 0xffff, /* nd == 6 */ 0x0000, /* nd == 7 reserved */ }; KASSERT(nd <= 6, ("number of domains %d", nd)); return (masks[nd]); } static const struct sagaw_bits_tag { int agaw; int cap; int awlvl; int pglvl; } sagaw_bits[] = { {.agaw = 30, .cap = DMAR_CAP_SAGAW_2LVL, .awlvl = DMAR_CTX2_AW_2LVL, .pglvl = 2}, {.agaw = 39, .cap = DMAR_CAP_SAGAW_3LVL, .awlvl = DMAR_CTX2_AW_3LVL, .pglvl = 3}, {.agaw = 48, .cap = DMAR_CAP_SAGAW_4LVL, .awlvl = DMAR_CTX2_AW_4LVL, .pglvl = 4}, {.agaw = 57, .cap = DMAR_CAP_SAGAW_5LVL, .awlvl = DMAR_CTX2_AW_5LVL, .pglvl = 5}, {.agaw = 64, .cap = DMAR_CAP_SAGAW_6LVL, .awlvl = DMAR_CTX2_AW_6LVL, .pglvl = 6} }; bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl) { int i; for (i = 0; i < nitems(sagaw_bits); i++) { if (sagaw_bits[i].pglvl != pglvl) continue; if ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) return (true); } return (false); } int domain_set_agaw(struct dmar_domain *domain, int mgaw) { int sagaw, i; domain->mgaw = mgaw; sagaw = DMAR_CAP_SAGAW(domain->dmar->hw_cap); for (i = 0; i < nitems(sagaw_bits); i++) { if (sagaw_bits[i].agaw >= mgaw) { domain->agaw = sagaw_bits[i].agaw; domain->pglvl = sagaw_bits[i].pglvl; domain->awlvl = sagaw_bits[i].awlvl; return (0); } } device_printf(domain->dmar->dev, "context request mgaw %d: no agaw found, sagaw %x\n", mgaw, sagaw); return (EINVAL); } /* * Find a best fit mgaw for the given maxaddr: * - if allow_less is false, must find sagaw which maps all requested * addresses (used by identity mappings); * - if allow_less is true, and no supported sagaw can map all requested * address space, accept the biggest sagaw, whatever is it. */ int dmar_maxaddr2mgaw(struct dmar_unit *unit, dmar_gaddr_t maxaddr, bool allow_less) { int i; for (i = 0; i < nitems(sagaw_bits); i++) { if ((1ULL << sagaw_bits[i].agaw) >= maxaddr && (DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) break; } if (allow_less && i == nitems(sagaw_bits)) { do { i--; } while ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) == 0); } if (i < nitems(sagaw_bits)) return (sagaw_bits[i].agaw); KASSERT(0, ("no mgaw for maxaddr %jx allow_less %d", (uintmax_t) maxaddr, allow_less)); return (-1); } /* * Calculate the total amount of page table pages needed to map the * whole bus address space on the context with the selected agaw. */ vm_pindex_t pglvl_max_pages(int pglvl) { vm_pindex_t res; int i; for (res = 0, i = pglvl; i > 0; i--) { res *= DMAR_NPTEPG; res++; } return (res); } /* * Return true if the page table level lvl supports the superpage for * the context ctx. */ int domain_is_sp_lvl(struct dmar_domain *domain, int lvl) { int alvl, cap_sps; static const int sagaw_sp[] = { DMAR_CAP_SPS_2M, DMAR_CAP_SPS_1G, DMAR_CAP_SPS_512G, DMAR_CAP_SPS_1T }; alvl = domain->pglvl - lvl - 1; cap_sps = DMAR_CAP_SPS(domain->dmar->hw_cap); return (alvl < nitems(sagaw_sp) && (sagaw_sp[alvl] & cap_sps) != 0); } dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl) { int rlvl; static const dmar_gaddr_t pg_sz[] = { (dmar_gaddr_t)DMAR_PAGE_SIZE, (dmar_gaddr_t)DMAR_PAGE_SIZE << DMAR_NPTEPGSHIFT, (dmar_gaddr_t)DMAR_PAGE_SIZE << (2 * DMAR_NPTEPGSHIFT), (dmar_gaddr_t)DMAR_PAGE_SIZE << (3 * DMAR_NPTEPGSHIFT), (dmar_gaddr_t)DMAR_PAGE_SIZE << (4 * DMAR_NPTEPGSHIFT), (dmar_gaddr_t)DMAR_PAGE_SIZE << (5 * DMAR_NPTEPGSHIFT) }; KASSERT(lvl >= 0 && lvl < total_pglvl, ("total %d lvl %d", total_pglvl, lvl)); rlvl = total_pglvl - lvl - 1; KASSERT(rlvl < nitems(pg_sz), ("sizeof pg_sz lvl %d", lvl)); return (pg_sz[rlvl]); } dmar_gaddr_t domain_page_size(struct dmar_domain *domain, int lvl) { return (pglvl_page_size(domain->pglvl, lvl)); } int calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size, dmar_gaddr_t *isizep) { dmar_gaddr_t isize; int am; for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) { isize = 1ULL << (am + DMAR_PAGE_SHIFT); if ((base & (isize - 1)) == 0 && size >= isize) break; if (am == 0) break; } *isizep = isize; return (am); } dmar_haddr_t dmar_high; int haw; int dmar_tbl_pagecnt; vm_page_t dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags) { vm_page_t m; int zeroed, aflags; zeroed = (flags & DMAR_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0; aflags = zeroed | VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP | ((flags & DMAR_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL : VM_ALLOC_NOWAIT); for (;;) { if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); m = vm_page_lookup(obj, idx); if ((flags & DMAR_PGF_NOALLOC) != 0 || m != NULL) { if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); break; } m = vm_page_alloc_contig(obj, idx, aflags, 1, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); if (m != NULL) { if (zeroed && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); atomic_add_int(&dmar_tbl_pagecnt, 1); break; } if ((flags & DMAR_PGF_WAITOK) == 0) break; } return (m); } void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags) { vm_page_t m; if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); - m = vm_page_lookup(obj, idx); + m = vm_page_grab(obj, idx, VM_ALLOC_NOCREAT); if (m != NULL) { vm_page_free(m); atomic_subtract_int(&dmar_tbl_pagecnt, 1); } if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); } void * dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, struct sf_buf **sf) { vm_page_t m; bool allocated; if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); m = vm_page_lookup(obj, idx); if (m == NULL && (flags & DMAR_PGF_ALLOC) != 0) { m = dmar_pgalloc(obj, idx, flags | DMAR_PGF_OBJL); allocated = true; } else allocated = false; if (m == NULL) { if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); return (NULL); } /* Sleepable allocations cannot fail. */ if ((flags & DMAR_PGF_WAITOK) != 0) VM_OBJECT_WUNLOCK(obj); sched_pin(); *sf = sf_buf_alloc(m, SFB_CPUPRIVATE | ((flags & DMAR_PGF_WAITOK) == 0 ? SFB_NOWAIT : 0)); if (*sf == NULL) { sched_unpin(); if (allocated) { VM_OBJECT_ASSERT_WLOCKED(obj); dmar_pgfree(obj, m->pindex, flags | DMAR_PGF_OBJL); } if ((flags & DMAR_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); return (NULL); } if ((flags & (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) == (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) VM_OBJECT_WLOCK(obj); else if ((flags & (DMAR_PGF_WAITOK | DMAR_PGF_OBJL)) == 0) VM_OBJECT_WUNLOCK(obj); return ((void *)sf_buf_kva(*sf)); } void dmar_unmap_pgtbl(struct sf_buf *sf) { sf_buf_free(sf); sched_unpin(); } static void dmar_flush_transl_to_ram(struct dmar_unit *unit, void *dst, size_t sz) { if (DMAR_IS_COHERENT(unit)) return; /* * If DMAR does not snoop paging structures accesses, flush * CPU cache to memory. */ pmap_force_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz); } void dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } /* * Load the root entry pointer into the hardware, busily waiting for * the completion. */ int dmar_load_root_entry_ptr(struct dmar_unit *unit) { vm_page_t root_entry; int error; /* * Access to the GCMD register must be serialized while the * command is submitted. */ DMAR_ASSERT_LOCKED(unit); VM_OBJECT_RLOCK(unit->ctx_obj); root_entry = vm_page_lookup(unit->ctx_obj, 0); VM_OBJECT_RUNLOCK(unit->ctx_obj); dmar_write8(unit, DMAR_RTADDR_REG, VM_PAGE_TO_PHYS(root_entry)); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SRTP); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_RTPS) != 0)); return (error); } /* * Globally invalidate the context entries cache, busily waiting for * the completion. */ int dmar_inv_ctx_glob(struct dmar_unit *unit) { int error; /* * Access to the CCMD register must be serialized while the * command is submitted. */ DMAR_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); /* * The DMAR_CCMD_ICC bit in the upper dword should be written * after the low dword write is completed. Amd64 * dmar_write8() does not have this issue, i386 dmar_write8() * writes the upper dword last. */ dmar_write8(unit, DMAR_CCMD_REG, DMAR_CCMD_ICC | DMAR_CCMD_CIRG_GLOB); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_CCMD_REG + 4) & DMAR_CCMD_ICC32) == 0)); return (error); } /* * Globally invalidate the IOTLB, busily waiting for the completion. */ int dmar_inv_iotlb_glob(struct dmar_unit *unit) { int error, reg; DMAR_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap); /* See a comment about DMAR_CCMD_ICC in dmar_inv_ctx_glob. */ dmar_write8(unit, reg + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT | DMAR_IOTLB_IIRG_GLB | DMAR_IOTLB_DR | DMAR_IOTLB_DW); DMAR_WAIT_UNTIL(((dmar_read4(unit, reg + DMAR_IOTLB_REG_OFF + 4) & DMAR_IOTLB_IVT32) == 0)); return (error); } /* * Flush the chipset write buffers. See 11.1 "Write Buffer Flushing" * in the architecture specification. */ int dmar_flush_write_bufs(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); /* * DMAR_GCMD_WBF is only valid when CAP_RWBF is reported. */ KASSERT((unit->hw_cap & DMAR_CAP_RWBF) != 0, ("dmar%d: no RWBF", unit->unit)); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_WBF); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_WBFS) != 0)); return (error); } int dmar_enable_translation(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) != 0)); return (error); } int dmar_disable_translation(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) == 0)); return (error); } int dmar_load_irt_ptr(struct dmar_unit *unit) { uint64_t irta, s; int error; DMAR_ASSERT_LOCKED(unit); irta = unit->irt_phys; if (DMAR_X2APIC(unit)) irta |= DMAR_IRTA_EIME; s = fls(unit->irte_cnt) - 2; KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK && powerof2(unit->irte_cnt), ("IRTA_REG_S overflow %x", unit->irte_cnt)); irta |= s; dmar_write8(unit, DMAR_IRTA_REG, irta); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SIRTP); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS) != 0)); return (error); } int dmar_enable_ir(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_IRE; unit->hw_gcmd &= ~DMAR_GCMD_CFI; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) != 0)); return (error); } int dmar_disable_ir(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_IRE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) == 0)); return (error); } #define BARRIER_F \ u_int f_done, f_inproc, f_wakeup; \ \ f_done = 1 << (barrier_id * 3); \ f_inproc = 1 << (barrier_id * 3 + 1); \ f_wakeup = 1 << (barrier_id * 3 + 2) bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id) { BARRIER_F; DMAR_LOCK(dmar); if ((dmar->barrier_flags & f_done) != 0) { DMAR_UNLOCK(dmar); return (false); } if ((dmar->barrier_flags & f_inproc) != 0) { while ((dmar->barrier_flags & f_inproc) != 0) { dmar->barrier_flags |= f_wakeup; msleep(&dmar->barrier_flags, &dmar->lock, 0, "dmarb", 0); } KASSERT((dmar->barrier_flags & f_done) != 0, ("dmar%d barrier %d missing done", dmar->unit, barrier_id)); DMAR_UNLOCK(dmar); return (false); } dmar->barrier_flags |= f_inproc; DMAR_UNLOCK(dmar); return (true); } void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id) { BARRIER_F; DMAR_ASSERT_LOCKED(dmar); KASSERT((dmar->barrier_flags & (f_done | f_inproc)) == f_inproc, ("dmar%d barrier %d missed entry", dmar->unit, barrier_id)); dmar->barrier_flags |= f_done; if ((dmar->barrier_flags & f_wakeup) != 0) wakeup(&dmar->barrier_flags); dmar->barrier_flags &= ~(f_inproc | f_wakeup); DMAR_UNLOCK(dmar); } int dmar_batch_coalesce = 100; struct timespec dmar_hw_timeout = { .tv_sec = 0, .tv_nsec = 1000000 }; static const uint64_t d = 1000000000; void dmar_update_timeout(uint64_t newval) { /* XXXKIB not atomic */ dmar_hw_timeout.tv_sec = newval / d; dmar_hw_timeout.tv_nsec = newval % d; } uint64_t dmar_get_timeout(void) { return ((uint64_t)dmar_hw_timeout.tv_sec * d + dmar_hw_timeout.tv_nsec); } static int dmar_timeout_sysctl(SYSCTL_HANDLER_ARGS) { uint64_t val; int error; val = dmar_get_timeout(); error = sysctl_handle_long(oidp, &val, 0, req); if (error != 0 || req->newptr == NULL) return (error); dmar_update_timeout(val); return (error); } static SYSCTL_NODE(_hw, OID_AUTO, dmar, CTLFLAG_RD, NULL, ""); SYSCTL_INT(_hw_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD, &dmar_tbl_pagecnt, 0, "Count of pages used for DMAR pagetables"); SYSCTL_INT(_hw_dmar, OID_AUTO, batch_coalesce, CTLFLAG_RWTUN, &dmar_batch_coalesce, 0, "Number of qi batches between interrupt"); SYSCTL_PROC(_hw_dmar, OID_AUTO, timeout, CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, dmar_timeout_sysctl, "QU", "Timeout for command wait, in nanoseconds"); #ifdef INVARIANTS int dmar_check_free; SYSCTL_INT(_hw_dmar, OID_AUTO, check_free, CTLFLAG_RWTUN, &dmar_check_free, 0, "Check the GPA RBtree for free_down and free_after validity"); #endif