Index: lib/Makefile =================================================================== --- lib/Makefile +++ lib/Makefile @@ -99,6 +99,7 @@ libugidfw \ libulog \ libutil \ + libuvmem \ ${_libvgl} \ libwrap \ libxo \ Index: lib/libuvmem/Makefile =================================================================== --- /dev/null +++ lib/libuvmem/Makefile @@ -0,0 +1,19 @@ +# $FreeBSD$ + +PACKAGE= runtime + +LIB= uvmem +SRCS= subr_vmem.c + +SHLIB_MAJOR = 1 +LIBADD+= pthread +CFLAGS.clang+=-Wno-thread-safety-analysis + +SYMBOL_MAPS= ${.CURDIR}/Symbol.map +VERSION_DEF= ${.CURDIR}/../libc/Versions.def + +.include + +.PATH: ${SRCTOP}/sys/kern + +.include Index: lib/libuvmem/Symbol.map =================================================================== --- /dev/null +++ lib/libuvmem/Symbol.map @@ -0,0 +1,21 @@ +/* $FreeBSD$ */ + +FBSD_1.6 { + vmem_add; + vmem_alloc; + vmem_create; + vmem_destroy; + vmem_free; + vmem_init; + vmem_print; + vmem_printall; + vmem_roundup_size; + vmem_set_import; + vmem_set_limit; + vmem_set_reclaim; + vmem_size; + vmem_startup; + vmem_whatis; + vmem_xalloc; + vmem_xfree; +}; Index: share/mk/bsd.libnames.mk =================================================================== --- share/mk/bsd.libnames.mk +++ share/mk/bsd.libnames.mk @@ -159,6 +159,7 @@ LIBUSBHID?= ${LIBDESTDIR}${LIBDIR_BASE}/libusbhid.a LIBUTIL?= ${LIBDESTDIR}${LIBDIR_BASE}/libutil.a LIBUUTIL?= ${LIBDESTDIR}${LIBDIR_BASE}/libuutil.a +LIBUVMEM?= ${LIBDESTDIR}${LIBDIR_BASE}/libuvmem.a LIBVGL?= ${LIBDESTDIR}${LIBDIR_BASE}/libvgl.a LIBVMMAPI?= ${LIBDESTDIR}${LIBDIR_BASE}/libvmmapi.a LIBWIND?= ${LIBDESTDIR}${LIBDIR_BASE}/libwind.a Index: share/mk/src.libnames.mk =================================================================== --- share/mk/src.libnames.mk +++ share/mk/src.libnames.mk @@ -193,6 +193,7 @@ usb \ usbhid \ util \ + uvmem \ uutil \ vmmapi \ wind \ @@ -386,6 +387,7 @@ _DP_ipf= kvm _DP_tpool= spl _DP_uutil= avl spl +_DP_uvmem= pthread _DP_zfs= md pthread umem util uutil m avl bsdxml crypto geom nvpair \ z zfs_core zutil _DP_zfsbootenv= zfs nvpair @@ -592,6 +594,9 @@ LIBC_NOSSP_PICDIR= ${_LIB_OBJTOP}/lib/libc LIBC_NOSSP_PIC?= ${LIBC_NOSSP_PICDIR}/libc_nossp_pic.a +LIBUVMEMDIR= ${OBJTOP}/lib/libuvmem +LIBUVMEM?= ${LIBUVMEMDIR}/libuvmem${PIE_SUFFIX}.a + # Define a directory for each library. This is useful for adding -L in when # not using a --sysroot or for meta mode bootstrapping when there is no # Makefile.depend. These are sorted by directory. Index: sys/contrib/openzfs/include/sys/zfs_context.h =================================================================== --- sys/contrib/openzfs/include/sys/zfs_context.h +++ sys/contrib/openzfs/include/sys/zfs_context.h @@ -50,7 +50,9 @@ #include #include #include +#ifndef __FreeBSD__ #include +#endif #include #include #include Index: sys/kern/subr_vmem.c =================================================================== --- sys/kern/subr_vmem.c +++ sys/kern/subr_vmem.c @@ -43,6 +43,8 @@ #include __FBSDID("$FreeBSD$"); +#ifdef _KERNEL + #include "opt_ddb.h" #include @@ -77,6 +79,28 @@ #include #include +#else /* _KERNEL */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KASSERT(a, b) +#define MPASS(a) +#define WITNESS_WARN(a, b, c) +#define panic(...) assert(0) + +#endif /* _KERNEL */ + #define VMEM_OPTORDER 5 #define VMEM_OPTVALUE (1 << VMEM_OPTORDER) #define VMEM_MAXORDER \ @@ -89,24 +113,42 @@ #define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) -#define VMEM_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | \ - M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) - -#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) - #define QC_NAME_MAX 16 /* * Data structures private to vmem. */ +#ifdef _KERNEL + +#define VMEM_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | \ + M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) + +#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) + MALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures"); +#else /* _KERNEL */ + +/* bit-compat with kernel */ +#define M_NOWAIT 0x0001 +#define M_WAITOK 0x0002 +#define M_ZERO 0 +#define M_NOVM 0 +#define M_USE_RESERVE 0 + +#define VMEM_FLAGS (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) + +#define BT_FLAGS 0 + +#endif /* _KERNEL */ + typedef struct vmem_btag bt_t; TAILQ_HEAD(vmem_seglist, vmem_btag); LIST_HEAD(vmem_freelist, vmem_btag); LIST_HEAD(vmem_hashlist, vmem_btag); +#ifdef _KERNEL struct qcache { uma_zone_t qc_cache; vmem_t *qc_vmem; @@ -115,6 +157,7 @@ }; typedef struct qcache qcache_t; #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache)) +#endif #define VMEM_NAME_MAX 16 @@ -134,8 +177,13 @@ /* vmem arena */ struct vmem { +#ifdef _KERNEL struct mtx_padalign vm_lock; struct cv vm_cv; +#else + pthread_mutex_t vm_lock; + pthread_cond_t vm_cv; +#endif char vm_name[VMEM_NAME_MAX+1]; LIST_ENTRY(vmem) vm_alllist; struct vmem_hashlist vm_hash0[VMEM_HASHSIZE_MIN]; @@ -167,8 +215,10 @@ /* Space exhaustion callback. */ vmem_reclaim_t *vm_reclaimfn; +#ifdef _KERNEL /* quantum cache */ qcache_t vm_qcache[VMEM_QCACHE_IDX_MAX]; +#endif }; #define BT_TYPE_SPAN 1 /* Allocated from importfn */ @@ -180,6 +230,7 @@ #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1) +#ifdef _KERNEL #if defined(DIAGNOSTIC) static int enable_vmem_check = 1; SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RWTUN, @@ -192,10 +243,17 @@ static struct task vmem_periodic_wk; static struct mtx_padalign __exclusive_cache_line vmem_list_lock; -static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); static uma_zone_t vmem_zone; +#else /* _KERNEL */ +static pthread_mutex_t vmem_list_lock = PTHREAD_MUTEX_INITIALIZER; + +#endif /* _KERNEL */ + +static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); + /* ---- misc */ +#ifdef _KERNEL #define VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan) #define VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv) #define VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock) @@ -207,6 +265,19 @@ #define VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF) #define VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock) #define VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED); +#else /* _KERNEL */ +#define VMEM_CONDVAR_INIT(vm, wchan) pthread_cond_init(&vm->vm_cv, NULL) +#define VMEM_CONDVAR_DESTROY(vm) pthread_cond_destroy(&vm->vm_cv) +#define VMEM_CONDVAR_WAIT(vm) pthread_cond_wait(&vm->vm_cv, &vm->vm_lock) +#define VMEM_CONDVAR_BROADCAST(vm) pthread_cond_broadcast(&vm->vm_cv) + +#define VMEM_LOCK(vm) pthread_mutex_lock(&vm->vm_lock) +#define VMEM_TRYLOCK(vm) pthread_mutex_trylock(&vm->vm_lock) +#define VMEM_UNLOCK(vm) pthread_mutex_unlock(&vm->vm_lock) +#define VMEM_LOCK_INIT(vm, name) pthread_mutex_init(&vm->vm_lock, NULL) +#define VMEM_LOCK_DESTROY(vm) pthread_mutex_destroy(&vm->vm_lock) +#define VMEM_ASSERT_LOCKED(vm) pthread_mutex_isowned_np(&vm->vm_lock) +#endif /* _KERNEL */ #define VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align))) @@ -231,6 +302,7 @@ */ #define BT_MAXFREE (BT_MAXALLOC * 8) +#ifdef _KERNEL /* Allocator for boundary tags. */ static uma_zone_t vmem_bt_zone; @@ -247,7 +319,8 @@ #ifdef DEBUG_MEMGUARD static struct vmem memguard_arena_storage; vmem_t *memguard_arena = &memguard_arena_storage; -#endif +#endif /* DEBUG_MEMGUARD */ +#endif /* _KERNEL */ static bool bt_isbusy(bt_t *bt) @@ -278,8 +351,10 @@ * dip into reserve tags. They are where new tags come from. */ flags &= BT_FLAGS; +#ifdef _KERNEL if (vm != kernel_arena && vm->vm_arg != kernel_arena) flags &= ~M_USE_RESERVE; +#endif /* * Loop until we meet the reserve. To minimize the lock shuffle @@ -288,12 +363,18 @@ * holding a vmem lock. */ while (vm->vm_nfreetags < BT_MAXALLOC) { +#ifdef _KERNEL bt = uma_zalloc(vmem_bt_zone, (flags & M_USE_RESERVE) | M_NOWAIT | M_NOVM); +#else + bt = malloc(sizeof(struct vmem_btag)); +#endif if (bt == NULL) { +#ifdef _KERNEL VMEM_UNLOCK(vm); bt = uma_zalloc(vmem_bt_zone, flags); VMEM_LOCK(vm); +#endif if (bt == NULL) break; } @@ -353,7 +434,11 @@ VMEM_UNLOCK(vm); while ((bt = LIST_FIRST(&freetags)) != NULL) { LIST_REMOVE(bt, bt_freelist); +#ifdef _KERNEL uma_zfree(vmem_bt_zone, bt); +#else + free(bt); +#endif } } @@ -540,6 +625,7 @@ /* ---- vmem internal functions */ +#ifdef _KERNEL /* * Import from the arena into the quantum cache in UMA. * @@ -728,8 +814,6 @@ #endif } -/* ---- rehash */ - static int vmem_rehash(vmem_t *vm, vmem_size_t newhashsize) { @@ -827,6 +911,7 @@ vmem_periodic_kick, NULL); } SYSINIT(vfs, SI_SUB_CONFIGURE, SI_ORDER_ANY, vmem_start_callout, NULL); +#endif /* _KERNEL */ static void vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type) @@ -882,10 +967,12 @@ { bt_t *bt; +#ifdef _KERNEL /* * Drain per-cpu quantum caches. */ qc_destroy(vm); +#endif /* * The vmem should now only contain empty segments. @@ -897,14 +984,23 @@ while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL) bt_remseg(vm, bt); - if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0) + if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0) { +#ifdef _KERNEL free(vm->vm_hashlist, M_VMEM); +#else + free(vm->vm_hashlist); +#endif + } bt_freetrim(vm, 0); VMEM_CONDVAR_DESTROY(vm); VMEM_LOCK_DESTROY(vm); +#ifdef _KERNEL uma_zfree(vmem_zone, vm); +#else + free(vm); +#endif } static int @@ -1061,8 +1157,10 @@ avail = vm->vm_size - vm->vm_inuse; bt_save(vm); VMEM_UNLOCK(vm); +#ifdef _KERNEL if (vm->vm_qcache_max != 0) qc_drain(vm); +#endif if (vm->vm_reclaimfn != NULL) vm->vm_reclaimfn(vm, flags); VMEM_LOCK(vm); @@ -1242,8 +1340,13 @@ { vmem_size_t i; +#ifdef _KERNEL MPASS(quantum > 0); MPASS((quantum & (quantum - 1)) == 0); +#else + assert(quantum == 0); + assert(qcache_max == 0); +#endif bzero(vm, sizeof(*vm)); @@ -1258,7 +1361,11 @@ vm->vm_size = 0; vm->vm_limit = 0; vm->vm_inuse = 0; +#ifdef _KERNEL qc_init(vm, qcache_max); +#else + qcache_max++; +#endif TAILQ_INIT(&vm->vm_seglist); vm->vm_cursor.bt_start = vm->vm_cursor.bt_size = 0; @@ -1279,9 +1386,17 @@ } } +#ifdef _KERNEL mtx_lock(&vmem_list_lock); +#else + pthread_mutex_lock(&vmem_list_lock); +#endif LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); +#ifdef _KERNEL mtx_unlock(&vmem_list_lock); +#else + pthread_mutex_unlock(&vmem_list_lock); +#endif return vm; } @@ -1296,7 +1411,13 @@ vmem_t *vm; +#ifdef _KERNEL vm = uma_zalloc(vmem_zone, flags & (M_WAITOK|M_NOWAIT)); +#else + assert(quantum == 0); + assert(qcache_max == 0); + vm = malloc(sizeof(vmem_t)); +#endif if (vm == NULL) return (NULL); if (vmem_init(vm, name, base, size, quantum, qcache_max, @@ -1308,10 +1429,17 @@ void vmem_destroy(vmem_t *vm) { - +#ifdef _KERNEL mtx_lock(&vmem_list_lock); +#else + pthread_mutex_lock(&vmem_list_lock); +#endif LIST_REMOVE(vm, vm_alllist); +#ifdef _KERNEL mtx_unlock(&vmem_list_lock); +#else + pthread_mutex_unlock(&vmem_list_lock); +#endif vmem_destroy1(vm); } @@ -1330,7 +1458,6 @@ vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp) { const int strat __unused = flags & VMEM_FITMASK; - qcache_t *qc; flags &= VMEM_FLAGS; MPASS(size > 0); @@ -1338,7 +1465,10 @@ if ((flags & M_NOWAIT) == 0) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_alloc"); +#ifdef _KERNEL if (size <= vm->vm_qcache_max) { + qcache_t *qc; + /* * Resource 0 cannot be cached, so avoid a blocking allocation * in qc_import() and give the vmem_xalloc() call below a chance @@ -1350,6 +1480,7 @@ if (__predict_true(*addrp != 0)) return (0); } +#endif return (vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags, addrp)); @@ -1469,14 +1600,17 @@ void vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) { - qcache_t *qc; MPASS(size > 0); +#ifdef _KERNEL if (size <= vm->vm_qcache_max && __predict_true(addr >= VMEM_ADDR_QCACHE_MIN)) { + qcache_t *qc; + qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift]; uma_zfree(qc->qc_cache, (void *)addr); } else +#endif vmem_xfree(vm, addr, size); } @@ -1570,11 +1704,13 @@ return (0); default: panic("vmem_size"); + return (0); } } /* ---- debug */ +#ifdef _KERNEL #if defined(DDB) || defined(DIAGNOSTIC) static void bt_dump(const bt_t *, int (*)(const char *, ...) @@ -1826,3 +1962,4 @@ } #endif /* defined(DIAGNOSTIC) */ +#endif /* _KERNEL */ Index: sys/sys/vmem.h =================================================================== --- sys/sys/vmem.h +++ sys/sys/vmem.h @@ -34,8 +34,6 @@ #include -#ifdef _KERNEL - typedef struct vmem vmem_t; typedef uintptr_t vmem_addr_t; @@ -45,10 +43,23 @@ #define VMEM_ADDR_QCACHE_MIN 1 #define VMEM_ADDR_MAX (~(vmem_addr_t)0) +/* vmem_size typemask */ +#define VMEM_ALLOC 0x01 +#define VMEM_FREE 0x02 +#define VMEM_MAXFREE 0x10 + typedef int (vmem_import_t)(void *, vmem_size_t, int, vmem_addr_t *); typedef void (vmem_release_t)(void *, vmem_addr_t, vmem_size_t); typedef void (vmem_reclaim_t)(vmem_t *, int); +#ifndef _KERNEL +#define M_FIRSTFIT 0x1000 /* only for vmem, fast fit */ +#define M_BESTFIT 0x2000 /* only for vmem, low fragmentation */ +#define M_NEXTFIT 0x8000 /* only for vmem, follow cursor */ +#endif + +__BEGIN_DECLS + /* * Create a vmem: * name - Name of the region @@ -135,11 +146,6 @@ __printflike(1, 2)); void vmem_startup(void); -/* vmem_size typemask */ -#define VMEM_ALLOC 0x01 -#define VMEM_FREE 0x02 -#define VMEM_MAXFREE 0x10 - -#endif /* _KERNEL */ +__END_DECLS #endif /* !_SYS_VMEM_H_ */ Index: usr.sbin/bhyve/Makefile =================================================================== --- usr.sbin/bhyve/Makefile +++ usr.sbin/bhyve/Makefile @@ -84,7 +84,7 @@ .PATH: ${BHYVE_SYSDIR}/sys/amd64/vmm SRCS+= vmm_instruction_emul.c -LIBADD= vmmapi md pthread z util sbuf cam 9p casper cap_pwd cap_grp +LIBADD= vmmapi md pthread z util sbuf cam 9p casper cap_pwd cap_grp uvmem .if ${MK_BHYVE_SNAPSHOT} != "no" LIBADD+= ucl xo .endif