diff --git a/lib/Makefile b/lib/Makefile --- a/lib/Makefile +++ b/lib/Makefile @@ -107,6 +107,7 @@ libulog \ libutil \ libutil++ \ + libuvmem \ ${_libvgl} \ libwrap \ libxo \ diff --git a/lib/libuvmem/Makefile b/lib/libuvmem/Makefile new file mode 100644 --- /dev/null +++ b/lib/libuvmem/Makefile @@ -0,0 +1,18 @@ + +PACKAGE= runtime + +LIB= uvmem +SRCS= subr_vmem.c + +SHLIB_MAJOR = 1 +LIBADD+= pthread +CFLAGS.clang+=-Wno-thread-safety-analysis + +SYMBOL_MAPS= ${.CURDIR}/Symbol.map +VERSION_DEF= ${SRCTOP}/lib/libc/Versions.def + +.include + +.PATH: ${SRCTOP}/sys/kern + +.include diff --git a/lib/libuvmem/Symbol.map b/lib/libuvmem/Symbol.map new file mode 100644 --- /dev/null +++ b/lib/libuvmem/Symbol.map @@ -0,0 +1,15 @@ +FBSD_1.9 { + vmem_add; + vmem_alloc; + vmem_create; + vmem_destroy; + vmem_free; + vmem_init; + vmem_roundup_size; + vmem_set_import; + vmem_set_limit; + vmem_set_reclaim; + vmem_size; + vmem_xalloc; + vmem_xfree; +}; diff --git a/share/mk/bsd.libnames.mk b/share/mk/bsd.libnames.mk --- a/share/mk/bsd.libnames.mk +++ b/share/mk/bsd.libnames.mk @@ -171,6 +171,7 @@ LIBUSBHID?= ${LIBDESTDIR}${LIBDIR_BASE}/libusbhid.a LIBUTIL?= ${LIBDESTDIR}${LIBDIR_BASE}/libutil.a LIBUUTIL?= ${LIBDESTDIR}${LIBDIR_BASE}/libuutil.a +LIBUVMEM?= ${LIBDESTDIR}${LIBDIR_BASE}/libuvmem.a LIBVERTO?= ${LIBDESTDIR}${LIBDIR_BASE}/libverto.a LIBVGL?= ${LIBDESTDIR}${LIBDIR_BASE}/libvgl.a LIBVMMAPI?= ${LIBDESTDIR}${LIBDIR_BASE}/libvmmapi.a diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk --- a/share/mk/src.libnames.mk +++ b/share/mk/src.libnames.mk @@ -236,6 +236,7 @@ usb \ usbhid \ util \ + uvmem \ uutil \ verto \ vmmapi \ @@ -490,6 +491,7 @@ _DP_ipf= kvm _DP_tpool= spl _DP_uutil= avl spl +_DP_uvmem= pthread _DP_zfs= md pthread rt umem util uutil m avl bsdxml crypto geom nvpair \ z zfs_core zutil _DP_zfsbootenv= zfs nvpair @@ -766,6 +768,9 @@ LIBSAMPLERATEDIR?= ${_LIB_OBJTOP}/lib/libsamplerate LIBSAMPLERATE?= ${LIBSAMPLERATEDIR}/libsamplerate${PIE_SUFFIX}.a +LIBUVMEMDIR= ${OBJTOP}/lib/libuvmem +LIBUVMEM?= ${LIBUVMEMDIR}/libuvmem${PIE_SUFFIX}.a + # Define a directory for each library. This is useful for adding -L in when # not using a --sysroot or for meta mode bootstrapping when there is no # Makefile.depend. These are sorted by directory. diff --git a/sys/contrib/openzfs/include/sys/zfs_context.h b/sys/contrib/openzfs/include/sys/zfs_context.h --- a/sys/contrib/openzfs/include/sys/zfs_context.h +++ b/sys/contrib/openzfs/include/sys/zfs_context.h @@ -50,7 +50,9 @@ #include #include #include +#ifndef __FreeBSD__ #include +#endif #include #include #include diff --git a/sys/kern/subr_vmem.c b/sys/kern/subr_vmem.c --- a/sys/kern/subr_vmem.c +++ b/sys/kern/subr_vmem.c @@ -41,6 +41,9 @@ */ #include + +#ifdef _KERNEL + #include "opt_ddb.h" #include @@ -75,6 +78,28 @@ #include #include +#else /* _KERNEL */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define KASSERT(a, b) +#define MPASS(a) +#define WITNESS_WARN(a, b, c) +#define panic(...) assert(0) + +#endif /* _KERNEL */ + #define VMEM_OPTORDER 5 #define VMEM_OPTVALUE (1 << VMEM_OPTORDER) #define VMEM_MAXORDER \ @@ -87,24 +112,42 @@ #define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) -#define VMEM_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | \ - M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) - -#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) - #define QC_NAME_MAX 16 /* * Data structures private to vmem. */ +#ifdef _KERNEL + +#define VMEM_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | \ + M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) + +#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM) + MALLOC_DEFINE(M_VMEM, "vmem", "vmem internal structures"); +#else /* _KERNEL */ + +/* bit-compat with kernel */ +#define M_NOWAIT 0x0001 +#define M_WAITOK 0x0002 +#define M_ZERO 0 +#define M_NOVM 0 +#define M_USE_RESERVE 0 + +#define VMEM_FLAGS (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT) + +#define BT_FLAGS 0 + +#endif /* _KERNEL */ + typedef struct vmem_btag bt_t; TAILQ_HEAD(vmem_seglist, vmem_btag); LIST_HEAD(vmem_freelist, vmem_btag); LIST_HEAD(vmem_hashlist, vmem_btag); +#ifdef _KERNEL struct qcache { uma_zone_t qc_cache; vmem_t *qc_vmem; @@ -113,6 +156,7 @@ }; typedef struct qcache qcache_t; #define QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache)) +#endif #define VMEM_NAME_MAX 16 @@ -132,8 +176,13 @@ /* vmem arena */ struct vmem { +#ifdef _KERNEL struct mtx_padalign vm_lock; struct cv vm_cv; +#else + pthread_mutex_t vm_lock; + pthread_cond_t vm_cv; +#endif char vm_name[VMEM_NAME_MAX+1]; LIST_ENTRY(vmem) vm_alllist; struct vmem_hashlist vm_hash0[VMEM_HASHSIZE_MIN]; @@ -165,8 +214,10 @@ /* Space exhaustion callback. */ vmem_reclaim_t *vm_reclaimfn; +#ifdef _KERNEL /* quantum cache */ qcache_t vm_qcache[VMEM_QCACHE_IDX_MAX]; +#endif }; #define BT_TYPE_SPAN 1 /* Allocated from importfn */ @@ -178,6 +229,7 @@ #define BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1) +#ifdef _KERNEL #if defined(DIAGNOSTIC) static int enable_vmem_check = 0; SYSCTL_INT(_debug, OID_AUTO, vmem_check, CTLFLAG_RWTUN, @@ -190,10 +242,20 @@ static struct task vmem_periodic_wk; static struct mtx_padalign __exclusive_cache_line vmem_list_lock; -static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); static uma_zone_t vmem_zone; +#else /* _KERNEL */ +static pthread_mutex_t vmem_list_lock = PTHREAD_MUTEX_INITIALIZER; + +#endif /* _KERNEL */ + +static LIST_HEAD(, vmem) vmem_list = LIST_HEAD_INITIALIZER(vmem_list); + /* ---- misc */ +#ifdef _KERNEL +#define VMEM_LIST_LOCK() mtx_lock(&vmem_list_lock) +#define VMEM_LIST_UNLOCK() mtx_unlock(&vmem_list_lock) + #define VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan) #define VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv) #define VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock) @@ -205,6 +267,22 @@ #define VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF) #define VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock) #define VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED); +#else /* _KERNEL */ +#define VMEM_LIST_LOCK() pthread_mutex_lock(&vmem_list_lock) +#define VMEM_LIST_UNLOCK() pthread_mutex_unlock(&vmem_list_lock) + +#define VMEM_CONDVAR_INIT(vm, wchan) pthread_cond_init(&vm->vm_cv, NULL) +#define VMEM_CONDVAR_DESTROY(vm) pthread_cond_destroy(&vm->vm_cv) +#define VMEM_CONDVAR_WAIT(vm) pthread_cond_wait(&vm->vm_cv, &vm->vm_lock) +#define VMEM_CONDVAR_BROADCAST(vm) pthread_cond_broadcast(&vm->vm_cv) + +#define VMEM_LOCK(vm) pthread_mutex_lock(&vm->vm_lock) +#define VMEM_TRYLOCK(vm) pthread_mutex_trylock(&vm->vm_lock) +#define VMEM_UNLOCK(vm) pthread_mutex_unlock(&vm->vm_lock) +#define VMEM_LOCK_INIT(vm, name) pthread_mutex_init(&vm->vm_lock, NULL) +#define VMEM_LOCK_DESTROY(vm) pthread_mutex_destroy(&vm->vm_lock) +#define VMEM_ASSERT_LOCKED(vm) pthread_mutex_isowned_np(&vm->vm_lock) +#endif /* _KERNEL */ #define VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align))) @@ -229,6 +307,7 @@ */ #define BT_MAXFREE (BT_MAXALLOC * 8) +#ifdef _KERNEL /* Allocator for boundary tags. */ static uma_zone_t vmem_bt_zone; @@ -243,7 +322,8 @@ #ifdef DEBUG_MEMGUARD static struct vmem memguard_arena_storage; vmem_t *memguard_arena = &memguard_arena_storage; -#endif +#endif /* DEBUG_MEMGUARD */ +#endif /* _KERNEL */ static bool bt_isbusy(bt_t *bt) @@ -263,12 +343,13 @@ * at least the maximum possible tag allocations in the arena. */ static __noinline int -_bt_fill(vmem_t *vm, int flags) +_bt_fill(vmem_t *vm, int flags __unused) { bt_t *bt; VMEM_ASSERT_LOCKED(vm); +#ifdef _KERNEL /* * Only allow the kernel arena and arenas derived from kernel arena to * dip into reserve tags. They are where new tags come from. @@ -276,6 +357,7 @@ flags &= BT_FLAGS; if (vm != kernel_arena && vm->vm_arg != kernel_arena) flags &= ~M_USE_RESERVE; +#endif /* * Loop until we meet the reserve. To minimize the lock shuffle @@ -284,12 +366,18 @@ * holding a vmem lock. */ while (vm->vm_nfreetags < BT_MAXALLOC) { +#ifdef _KERNEL bt = uma_zalloc(vmem_bt_zone, (flags & M_USE_RESERVE) | M_NOWAIT | M_NOVM); +#else + bt = malloc(sizeof(struct vmem_btag)); +#endif if (bt == NULL) { +#ifdef _KERNEL VMEM_UNLOCK(vm); bt = uma_zalloc(vmem_bt_zone, flags); VMEM_LOCK(vm); +#endif if (bt == NULL) break; } @@ -349,7 +437,11 @@ VMEM_UNLOCK(vm); while ((bt = LIST_FIRST(&freetags)) != NULL) { LIST_REMOVE(bt, bt_freelist); +#ifdef _KERNEL uma_zfree(vmem_bt_zone, bt); +#else + free(bt); +#endif } } @@ -536,6 +628,7 @@ /* ---- vmem internal functions */ +#ifdef _KERNEL /* * Import from the arena into the quantum cache in UMA. * @@ -683,7 +776,7 @@ vm_wait_domain(domain); return (NULL); } - mtx_unlock(&vmem_bt_lock); + mtx_lock(&vmem_bt_unlock); /* * We're either out of address space or lost a fill race. */ @@ -720,8 +813,6 @@ #endif } -/* ---- rehash */ - static int vmem_rehash(vmem_t *vm, vmem_size_t newhashsize) { @@ -777,7 +868,7 @@ vmem_size_t desired; vmem_size_t current; - mtx_lock(&vmem_list_lock); + VMEM_LIST_LOCK(); LIST_FOREACH(vm, &vmem_list, vm_alllist) { #ifdef DIAGNOSTIC /* Convenient time to verify vmem state. */ @@ -802,7 +893,7 @@ */ VMEM_CONDVAR_BROADCAST(vm); } - mtx_unlock(&vmem_list_lock); + VMEM_LIST_UNLOCK(); callout_reset(&vmem_periodic_ch, vmem_periodic_interval, vmem_periodic_kick, NULL); @@ -819,6 +910,7 @@ vmem_periodic_kick, NULL); } SYSINIT(vfs, SI_SUB_CONFIGURE, SI_ORDER_ANY, vmem_start_callout, NULL); +#endif /* _KERNEL */ static void vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type) @@ -874,10 +966,12 @@ { bt_t *bt; +#ifdef _KERNEL /* * Drain per-cpu quantum caches. */ qc_destroy(vm); +#endif /* * The vmem should now only contain empty segments. @@ -889,14 +983,23 @@ while ((bt = TAILQ_FIRST(&vm->vm_seglist)) != NULL) bt_remseg(vm, bt); - if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0) + if (vm->vm_hashlist != NULL && vm->vm_hashlist != vm->vm_hash0) { +#ifdef _KERNEL free(vm->vm_hashlist, M_VMEM); +#else + free(vm->vm_hashlist); +#endif + } bt_freetrim(vm, 0); VMEM_CONDVAR_DESTROY(vm); VMEM_LOCK_DESTROY(vm); +#ifdef _KERNEL uma_zfree(vmem_zone, vm); +#else + free(vm); +#endif } static int @@ -1053,8 +1156,10 @@ avail = vm->vm_size - vm->vm_inuse; bt_save(vm); VMEM_UNLOCK(vm); +#ifdef _KERNEL if (vm->vm_qcache_max != 0) qc_drain(vm); +#endif if (vm->vm_reclaimfn != NULL) vm->vm_reclaimfn(vm, flags); VMEM_LOCK(vm); @@ -1234,8 +1339,13 @@ { vmem_size_t i; +#ifdef _KERNEL MPASS(quantum > 0); MPASS((quantum & (quantum - 1)) == 0); +#else + assert(quantum == 0); + assert(qcache_max == 0); +#endif bzero(vm, sizeof(*vm)); @@ -1250,7 +1360,11 @@ vm->vm_size = 0; vm->vm_limit = 0; vm->vm_inuse = 0; +#ifdef _KERNEL qc_init(vm, qcache_max); +#else + (void)qcache_max; +#endif TAILQ_INIT(&vm->vm_seglist); vm->vm_cursor.bt_start = vm->vm_cursor.bt_size = 0; @@ -1271,9 +1385,9 @@ } } - mtx_lock(&vmem_list_lock); + VMEM_LIST_LOCK(); LIST_INSERT_HEAD(&vmem_list, vm, vm_alllist); - mtx_unlock(&vmem_list_lock); + VMEM_LIST_UNLOCK(); return vm; } @@ -1288,7 +1402,13 @@ vmem_t *vm; +#ifdef _KERNEL vm = uma_zalloc(vmem_zone, flags & (M_WAITOK|M_NOWAIT)); +#else + assert(quantum == 0); + assert(qcache_max == 0); + vm = malloc(sizeof(vmem_t)); +#endif if (vm == NULL) return (NULL); if (vmem_init(vm, name, base, size, quantum, qcache_max, @@ -1300,10 +1420,9 @@ void vmem_destroy(vmem_t *vm) { - - mtx_lock(&vmem_list_lock); + VMEM_LIST_LOCK(); LIST_REMOVE(vm, vm_alllist); - mtx_unlock(&vmem_list_lock); + VMEM_LIST_UNLOCK(); vmem_destroy1(vm); } @@ -1322,7 +1441,6 @@ vmem_alloc(vmem_t *vm, vmem_size_t size, int flags, vmem_addr_t *addrp) { const int strat __unused = flags & VMEM_FITMASK; - qcache_t *qc; flags &= VMEM_FLAGS; MPASS(size > 0); @@ -1330,7 +1448,10 @@ if ((flags & M_NOWAIT) == 0) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "vmem_alloc"); +#ifdef _KERNEL if (size <= vm->vm_qcache_max) { + qcache_t *qc; + /* * Resource 0 cannot be cached, so avoid a blocking allocation * in qc_import() and give the vmem_xalloc() call below a chance @@ -1342,6 +1463,7 @@ if (__predict_true(*addrp != 0)) return (0); } +#endif return (vmem_xalloc(vm, size, 0, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags, addrp)); @@ -1461,14 +1583,17 @@ void vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size) { - qcache_t *qc; MPASS(size > 0); +#ifdef _KERNEL if (size <= vm->vm_qcache_max && __predict_true(addr >= VMEM_ADDR_QCACHE_MIN)) { + qcache_t *qc; + qc = &vm->vm_qcache[(size - 1) >> vm->vm_quantum_shift]; uma_zfree(qc->qc_cache, (void *)addr); } else +#endif vmem_xfree(vm, addr, size); } @@ -1562,11 +1687,13 @@ return (0); default: panic("vmem_size"); + return (0); } } /* ---- debug */ +#ifdef _KERNEL #if defined(DDB) || defined(DIAGNOSTIC) static void bt_dump(const bt_t *, int (*)(const char *, ...) @@ -1818,3 +1945,4 @@ } #endif /* defined(DIAGNOSTIC) */ +#endif /* _KERNEL */ diff --git a/sys/sys/vmem.h b/sys/sys/vmem.h --- a/sys/sys/vmem.h +++ b/sys/sys/vmem.h @@ -33,8 +33,6 @@ #include -#ifdef _KERNEL - typedef struct vmem vmem_t; typedef uintptr_t vmem_addr_t; @@ -44,10 +42,23 @@ #define VMEM_ADDR_QCACHE_MIN 1 #define VMEM_ADDR_MAX (~(vmem_addr_t)0) +/* vmem_size typemask */ +#define VMEM_ALLOC 0x01 +#define VMEM_FREE 0x02 +#define VMEM_MAXFREE 0x10 + typedef int (vmem_import_t)(void *, vmem_size_t, int, vmem_addr_t *); typedef void (vmem_release_t)(void *, vmem_addr_t, vmem_size_t); typedef void (vmem_reclaim_t)(vmem_t *, int); +#ifndef _KERNEL +#define M_FIRSTFIT 0x1000 /* only for vmem, fast fit */ +#define M_BESTFIT 0x2000 /* only for vmem, low fragmentation */ +#define M_NEXTFIT 0x8000 /* only for vmem, follow cursor */ +#endif + +__BEGIN_DECLS + /* * Create a vmem: * name - Name of the region @@ -134,11 +145,6 @@ __printflike(1, 2)); void vmem_startup(void); -/* vmem_size typemask */ -#define VMEM_ALLOC 0x01 -#define VMEM_FREE 0x02 -#define VMEM_MAXFREE 0x10 - -#endif /* _KERNEL */ +__END_DECLS #endif /* !_SYS_VMEM_H_ */ diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile --- a/usr.sbin/bhyve/Makefile +++ b/usr.sbin/bhyve/Makefile @@ -95,7 +95,7 @@ -I${.CURDIR}/../../contrib/lib9p \ -I${SRCTOP}/sys -LIBADD+= vmmapi md nv pthread z util sbuf cam 9p +LIBADD+= vmmapi md nv uvmem pthread z util sbuf cam 9p .if ${MK_BHYVE_SNAPSHOT} != "no" LIBADD+= ucl xo